Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

fix: qualify vector page identities

nash_su697ed6137879
diff --git a/src-tauri/src/api_server.rs b/src-tauri/src/api_server.rs
index 7e23b1fb..a64b82d5 100644
--- a/src-tauri/src/api_server.rs
+++ b/src-tauri/src/api_server.rs
@@ -1888,7 +1888,6 @@ fn handle_embed_page(app: &AppHandle, project_id: &str, body: &str) -> ApiRespon
commands::page_embedding::PageEmbeddingErrorKind::NotFound => 404,
commands::page_embedding::PageEmbeddingErrorKind::Provider => 502,
commands::page_embedding::PageEmbeddingErrorKind::Storage => 500,
- commands::page_embedding::PageEmbeddingErrorKind::Conflict => 409,
commands::page_embedding::PageEmbeddingErrorKind::Timeout => 504,
};
err(status, error.message)
diff --git a/src-tauri/src/commands/page_embedding.rs b/src-tauri/src/commands/page_embedding.rs
index 6524698b..2fa51a7e 100644
--- a/src-tauri/src/commands/page_embedding.rs
+++ b/src-tauri/src/commands/page_embedding.rs
@@ -39,7 +39,6 @@ pub enum PageEmbeddingErrorKind {
NotFound,
Provider,
Storage,
- Conflict,
Timeout,
}
@@ -105,21 +104,17 @@ pub async fn embed_wiki_page(
)
})?;
- let page_id = page_path
- .file_stem()
- .and_then(|value| value.to_str())
- .filter(|value| !value.is_empty())
- .ok_or_else(|| {
- PageEmbeddingError::new(
- PageEmbeddingErrorKind::InvalidRequest,
- "Invalid wiki page name",
- )
- })?
- .to_string();
+ let page_id = vector_page_id(&normalized_path).ok_or_else(|| {
+ PageEmbeddingError::new(
+ PageEmbeddingErrorKind::InvalidRequest,
+ "Invalid wiki page name",
+ )
+ })?;
vectorstore::validate_page_id_for_v2(&page_id)
.map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::InvalidRequest, err))?;
+ let page_stem = page_id.rsplit('/').next().unwrap_or(&page_id);
if matches!(
- page_id.to_ascii_lowercase().as_str(),
+ page_stem.to_ascii_lowercase().as_str(),
"index" | "log" | "overview"
) {
return Err(PageEmbeddingError::new(
@@ -127,8 +122,6 @@ pub async fn embed_wiki_page(
"Aggregate wiki pages index.md, log.md, and overview.md are maintained by the app and are not vector-indexed",
));
}
- ensure_unique_page_stem(project_path, &page_path, &page_id)?;
-
let revision = format!("sha256:{:x}", Sha256::digest(content.as_bytes()));
let fingerprint = embedding_fingerprint(&revision, &config);
if !force {
@@ -195,6 +188,7 @@ pub async fn embed_wiki_page(
vectorstore::vector_upsert_chunks_with_revision(project_path, &page_id, rows, &fingerprint)
.await
.map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::Storage, err))?;
+ remove_legacy_vector_id_if_safe(project_path, &page_id).await;
Ok(PageEmbeddingResult {
path: normalized_path,
page_id,
@@ -285,18 +279,22 @@ fn embedding_fingerprint(revision: &str, config: &SearchEmbeddingConfig) -> Stri
)
}
-fn ensure_unique_page_stem(
- project_path: &str,
- page_path: &Path,
- page_id: &str,
-) -> Result<(), PageEmbeddingError> {
- let wiki_root = fs::canonicalize(Path::new(project_path).join("wiki")).map_err(|err| {
- PageEmbeddingError::new(
- PageEmbeddingErrorKind::NotFound,
- format!("Failed to resolve project wiki directory: {err}"),
- )
- })?;
- let mut collisions = WalkDir::new(wiki_root)
+fn vector_page_id(normalized_path: &str) -> Option<String> {
+ let normalized = normalized_path.replace('\\', "/");
+ let relative = normalized.strip_prefix("wiki/")?;
+ if relative.len() < 3 || !relative[relative.len() - 3..].eq_ignore_ascii_case(".md") {
+ return None;
+ }
+ let relative = relative[..relative.len() - 3].trim_matches('/').to_string();
+ (!relative.is_empty()).then_some(relative)
+}
+
+async fn remove_legacy_vector_id_if_safe(project_path: &str, page_id: &str) {
+ let Some(legacy_id) = page_id.rsplit('/').next().filter(|_| page_id.contains('/')) else {
+ return;
+ };
+ let wiki_root = Path::new(project_path).join("wiki");
+ let owners = WalkDir::new(&wiki_root)
.follow_links(false)
.into_iter()
.filter_map(Result::ok)
@@ -311,20 +309,18 @@ fn ensure_unique_page_stem(
.path()
.file_stem()
.and_then(|value| value.to_str())
- .is_some_and(|stem| stem.eq_ignore_ascii_case(page_id))
+ .is_some_and(|stem| stem.eq_ignore_ascii_case(legacy_id))
})
- .filter_map(|entry| fs::canonicalize(entry.path()).ok())
- .filter(|path| path != page_path);
- if let Some(collision) = collisions.next() {
- return Err(PageEmbeddingError::new(
- PageEmbeddingErrorKind::Conflict,
- format!(
- "Cannot index this page because another wiki page has the same filename stem: {}",
- collision.to_string_lossy()
- ),
- ));
+ .take(2)
+ .count();
+ if owners != 1 {
+ return;
+ }
+ if let Err(error) =
+ vectorstore::vector_delete_page(project_path.to_string(), legacy_id.to_string()).await
+ {
+ eprintln!("[Embedding] failed to remove legacy vector id '{legacy_id}': {error}");
}
- Ok(())
}
fn revision_path(project_path: &str, page_id: &str) -> PathBuf {
@@ -805,15 +801,20 @@ mod tests {
}
#[test]
- fn duplicate_stems_in_different_wiki_folders_are_rejected() {
- let root = project();
- let first = root.join("wiki/nested/page.md");
- fs::write(&first, "# First").unwrap();
- fs::create_dir_all(root.join("wiki/other")).unwrap();
- fs::write(root.join("wiki/other/page.md"), "# Second").unwrap();
- let error = ensure_unique_page_stem(root.to_str().unwrap(), &first, "page").unwrap_err();
- assert_eq!(error.kind, PageEmbeddingErrorKind::Conflict);
- let _ = fs::remove_dir_all(root);
+ fn vector_page_ids_include_the_wiki_relative_directory() {
+ assert_eq!(
+ vector_page_id("wiki/nested/page.md").as_deref(),
+ Some("nested/page")
+ );
+ assert_eq!(
+ vector_page_id("wiki/other/page.md").as_deref(),
+ Some("other/page")
+ );
+ assert_eq!(
+ vector_page_id("wiki/notes/Foo.MD").as_deref(),
+ Some("notes/Foo")
+ );
+ assert_eq!(vector_page_id("outside/page.md"), None);
}
#[test]
diff --git a/src-tauri/src/commands/search.rs b/src-tauri/src/commands/search.rs
index c1f29761..d3d935f1 100644
--- a/src-tauri/src/commands/search.rs
+++ b/src-tauri/src/commands/search.rs
@@ -343,7 +343,8 @@ pub async fn search_project_inner(
};
let query_phrase = trim_query_punctuation(&query.to_lowercase());
let mut results = Vec::new();
- let mut page_paths_by_stem = BTreeMap::new();
+ let mut page_paths_by_vector_id = BTreeMap::new();
+ let mut legacy_stem_candidates = BTreeMap::<String, Vec<String>>::new();
let mut graph_pages = BTreeMap::new();
let wiki_root = Path::new(&project_path).join("wiki");
@@ -351,7 +352,11 @@ pub async fn search_project_inner(
let mut searched_files = 0usize;
for entry in WalkDir::new(&wiki_root).into_iter().filter_map(Result::ok) {
if !entry.file_type().is_file()
- || entry.path().extension().and_then(|s| s.to_str()) != Some("md")
+ || !entry
+ .path()
+ .extension()
+ .and_then(|s| s.to_str())
+ .is_some_and(|ext| ext.eq_ignore_ascii_case("md"))
{
continue;
}
@@ -366,19 +371,16 @@ pub async fn search_project_inner(
Ok(content) => content,
Err(_) => continue,
};
+ let relative_path = relative_to_project(&project_path, entry.path());
+ if let Some(vector_id) = vector_page_id(&relative_path) {
+ page_paths_by_vector_id.insert(vector_id, relative_path.clone());
+ }
if let Some(stem) = entry.path().file_stem().and_then(|s| s.to_str()) {
- let previous = page_paths_by_stem.insert(
- stem.to_string(),
- relative_to_project(&project_path, entry.path()),
- );
- if let Some(previous) = previous {
- eprintln!(
- "[Search] duplicate wiki page stem '{stem}': '{previous}' and '{}' share one vector page_id",
- relative_to_project(&project_path, entry.path())
- );
- }
+ legacy_stem_candidates
+ .entry(stem.to_string())
+ .or_default()
+ .push(relative_path.clone());
}
- let relative_path = relative_to_project(&project_path, entry.path());
let title = extract_title(
&content,
entry
@@ -410,6 +412,10 @@ pub async fn search_project_inner(
}
}
}
+ let legacy_page_paths = legacy_stem_candidates
+ .into_iter()
+ .filter_map(|(stem, paths)| (paths.len() == 1).then(|| (stem, paths[0].clone())))
+ .collect::<BTreeMap<_, _>>();
let mut token_sorted = (0..results.len()).collect::<Vec<_>>();
token_sorted.sort_by(|a, b| {
@@ -439,7 +445,8 @@ pub async fn search_project_inner(
}
materialize_vector_only_results(
&vector_results,
- &page_paths_by_stem,
+ &page_paths_by_vector_id,
+ &legacy_page_paths,
&project_path,
&mut results,
include_content,
@@ -455,7 +462,13 @@ pub async fn search_project_inner(
}
if vector_hits > 0 {
- apply_rrf_scores(&mut results, &token_rank, &vector_rank, &vector_score);
+ apply_rrf_scores(
+ &mut results,
+ &token_rank,
+ &vector_rank,
+ &vector_score,
+ &legacy_page_paths,
+ );
}
results.sort_by(|a, b| {
@@ -486,10 +499,23 @@ fn apply_rrf_scores(
token_rank: &BTreeMap<String, usize>,
vector_rank: &BTreeMap<String, usize>,
vector_score: &BTreeMap<String, f32>,
+ legacy_page_paths: &BTreeMap<String, String>,
) {
for result in results {
let token = token_rank.get(&normalize_path(&result.path)).copied();
- let vector = vector_rank.get(&file_stem(&result.path)).copied();
+ let vector_id = vector_page_id(&result.path).unwrap_or_else(|| file_stem(&result.path));
+ let legacy_stem = file_stem(&result.path);
+ let legacy_matches_page = legacy_page_paths
+ .get(&legacy_stem)
+ .is_some_and(|path| normalize_path(path) == normalize_path(&result.path));
+ let vector = vector_rank
+ .get(&vector_id)
+ .or_else(|| {
+ legacy_matches_page
+ .then(|| vector_rank.get(&legacy_stem))
+ .flatten()
+ })
+ .copied();
let mut rrf = 0.0;
if let Some(rank) = token {
rrf += 1.0 / (RRF_K + rank as f64);
@@ -497,7 +523,15 @@ fn apply_rrf_scores(
if let Some(rank) = vector {
rrf += 1.0 / (RRF_K + rank as f64);
}
- if let Some(score) = vector_score.get(&file_stem(&result.path)).copied() {
+ if let Some(score) = vector_score
+ .get(&vector_id)
+ .or_else(|| {
+ legacy_matches_page
+ .then(|| vector_score.get(&legacy_stem))
+ .flatten()
+ })
+ .copied()
+ {
result.vector_score = Some(score);
}
result.score = rrf;
@@ -819,17 +853,35 @@ async fn search_by_embedding(
fn materialize_vector_only_results(
vector_results: &[PageVectorResult],
- page_paths_by_stem: &BTreeMap<String, String>,
+ page_paths_by_vector_id: &BTreeMap<String, String>,
+ legacy_page_paths: &BTreeMap<String, String>,
project_path: &str,
results: &mut Vec<ProjectSearchResult>,
include_content: bool,
) {
- let mut known: BTreeSet<String> = results.iter().map(|r| file_stem(&r.path)).collect();
- for vr in vector_results {
- if known.contains(&vr.id) {
- continue;
- }
- if let Some(rel) = page_paths_by_stem.get(&vr.id) {
+ let mut known_paths: BTreeSet<String> = results
+ .iter()
+ .map(|result| normalize_path(&result.path))
+ .collect();
+ // Materialize path-qualified rows first. During migration a page can have
+ // both a current row and a legacy basename row; the current score/snippet
+ // must win regardless of the backend's result order.
+ for qualified_only in [true, false] {
+ for vr in vector_results {
+ let rel = if qualified_only {
+ page_paths_by_vector_id.get(&vr.id)
+ } else if page_paths_by_vector_id.contains_key(&vr.id) {
+ None
+ } else {
+ legacy_page_paths.get(&vr.id)
+ };
+ let Some(rel) = rel else {
+ continue;
+ };
+ let normalized_rel = normalize_path(rel);
+ if known_paths.contains(&normalized_rel) {
+ continue;
+ }
let path = Path::new(project_path).join(rel);
let Ok(content) = fs::read_to_string(&path) else {
continue;
@@ -851,7 +903,7 @@ fn materialize_vector_only_results(
content: include_content.then_some(content),
graph_related_to: Vec::new(),
});
- known.insert(vr.id.clone());
+ known_paths.insert(normalized_rel);
}
}
}
@@ -1713,6 +1765,18 @@ fn file_stem(path: &str) -> String {
.to_string()
}
+fn vector_page_id(path: &str) -> Option<String> {
+ let normalized = normalize_path(path);
+ let wiki_relative = normalized.strip_prefix("wiki/").unwrap_or(&normalized);
+ if wiki_relative.len() < 3
+ || !wiki_relative[wiki_relative.len() - 3..].eq_ignore_ascii_case(".md")
+ {
+ return None;
+ }
+ let id = wiki_relative[..wiki_relative.len() - 3].trim_matches('/');
+ (!id.is_empty()).then(|| id.to_string())
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -2018,7 +2082,20 @@ mod tests {
let vector_score =
BTreeMap::from([("both".to_string(), 0.95), ("vector-only".to_string(), 0.8)]);
- apply_rrf_scores(&mut results, &token_rank, &vector_rank, &vector_score);
+ let legacy_page_paths = BTreeMap::from([
+ ("both".to_string(), "wiki/concepts/both.md".to_string()),
+ (
+ "vector-only".to_string(),
+ "wiki/concepts/vector-only.md".to_string(),
+ ),
+ ]);
+ apply_rrf_scores(
+ &mut results,
+ &token_rank,
+ &vector_rank,
+ &vector_score,
+ &legacy_page_paths,
+ );
results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap());
assert_eq!(results[0].path, "wiki/concepts/both.md");
@@ -2054,20 +2131,21 @@ mod tests {
"---\ntitle: Deep Page\n---\n\n# Deep Page\n\nThe literal query is absent here.",
);
let vector_results = vec![PageVectorResult {
- id: "deep-page".to_string(),
+ id: "custom/deep-page".to_string(),
score: 0.91,
chunk_text: "A semantic chunk explains the actual reason for retrieval.".to_string(),
heading_path: "Section > Detail".to_string(),
}];
let mut results = Vec::new();
let pages = BTreeMap::from([(
- "deep-page".to_string(),
+ "custom/deep-page".to_string(),
"wiki/custom/deep-page.md".to_string(),
)]);
materialize_vector_only_results(
&vector_results,
&pages,
+ &BTreeMap::new(),
&root.to_string_lossy(),
&mut results,
false,
@@ -2082,6 +2160,49 @@ mod tests {
let _ = fs::remove_dir_all(root);
}
+ #[test]
+ fn vector_only_materialization_prefers_qualified_rows_over_legacy_rows() {
+ let root = tmp_project();
+ write_page(&root, "wiki/entities/topic.md", "# Topic\n\nCurrent content.");
+ let vector_results = vec![
+ PageVectorResult {
+ id: "topic".to_string(),
+ score: 0.99,
+ chunk_text: "Stale legacy chunk".to_string(),
+ heading_path: String::new(),
+ },
+ PageVectorResult {
+ id: "entities/topic".to_string(),
+ score: 0.82,
+ chunk_text: "Current qualified chunk".to_string(),
+ heading_path: String::new(),
+ },
+ ];
+ let qualified = BTreeMap::from([(
+ "entities/topic".to_string(),
+ "wiki/entities/topic.md".to_string(),
+ )]);
+ let legacy = BTreeMap::from([(
+ "topic".to_string(),
+ "wiki/entities/topic.md".to_string(),
+ )]);
+ let mut results = Vec::new();
+
+ materialize_vector_only_results(
+ &vector_results,
+ &qualified,
+ &legacy,
+ &root.to_string_lossy(),
+ &mut results,
+ false,
+ );
+
+ assert_eq!(results.len(), 1);
+ assert_eq!(results[0].vector_score, Some(0.82));
+ assert!(results[0].snippet.contains("Current qualified chunk"));
+ let _ = fs::remove_dir_all(root);
+ }
+
#[test]
fn vector_snippet_empty_chunk_does_not_echo_query() {
let vector = PageVectorResult {
diff --git a/src-tauri/src/commands/vectorstore.rs b/src-tauri/src/commands/vectorstore.rs
index 8be032fe..7fbbc622 100644
--- a/src-tauri/src/commands/vectorstore.rs
+++ b/src-tauri/src/commands/vectorstore.rs
@@ -69,10 +69,10 @@ static VECTORSTORE_V2_LOCKS: OnceLock<Mutex<HashMap<String, Arc<tokio::sync::RwL
OnceLock::new();
/// Validate page_id to prevent filter/path injection without rejecting
-/// legitimate Unicode wiki filenames. Page ids are wiki file stems; CJK
-/// letters, spaces, and punctuation such as `·` / `:` / `(` are valid page
-/// names, but quotes and separators are unsafe because we interpolate page_id
-/// into LanceDB filters (`page_id = '...'`) and derive debug chunk ids from it.
+/// legitimate Unicode wiki filenames. Page ids are wiki-relative paths without
+/// `.md`; CJK letters, spaces, forward slashes, and punctuation such as `·` /
+/// `:` / `(` are valid. Quotes and backslashes remain unsafe because we
+/// interpolate page_id into LanceDB filters (`page_id = '...'`).
/// Format/invisible characters are rejected so visually identical ids cannot
/// differ only by soft hyphen, zero-width, bidi, tag, or separator characters.
fn validate_page_id_common(page_id: &str) -> Result<(), String> {
@@ -100,8 +100,7 @@ fn is_disallowed_page_id_char(c: char) -> bool {
c.is_control()
|| matches!(
c,
- '/' | '\\'
- | '\''
+ '\\' | '\''
| '"'
| '\u{00AD}'
| '\u{061C}'
@@ -985,7 +984,7 @@ mod tests_v2 {
#[test]
fn page_id_validation_allows_unicode_wiki_stems() {
- let page_id = "反硝化除磷·A2O:DPAO + 50% & x(测试),v1.2";
+ let page_id = "concepts/反硝化除磷·A2O:DPAO + 50% & x(测试),v1.2";
assert!(validate_page_id(page_id).is_ok());
assert!(validate_page_id_for_v2(page_id).is_ok());
}
@@ -995,7 +994,6 @@ mod tests_v2 {
for page_id in [
"bad'quote",
"bad\"quote",
- "bad/slash",
"bad\\slash",
"bad\nnewline",
"bad\ttab",
diff --git a/src/lib/deep-research.test.ts b/src/lib/deep-research.test.ts
index 4d86b232..ce261dbf 100644
--- a/src/lib/deep-research.test.ts
+++ b/src/lib/deep-research.test.ts
@@ -117,9 +117,9 @@ describe("makeAvailableResearchFilePath", () => {
it("derives the vector page id from the final collision-safe path", () => {
expect(researchPageIdFromPath(
"/project/wiki/queries/research-topic-2026-08-20-120000-research-41-2.md",
- )).toBe("research-topic-2026-08-20-120000-research-41-2")
+ )).toBe("queries/research-topic-2026-08-20-120000-research-41-2")
expect(researchPageIdFromPath("C:\\project\\wiki\\queries\\research-topic-3.MD"))
- .toBe("research-topic-3")
+ .toBe("queries/research-topic-3")
})
})
diff --git a/src/lib/deep-research.ts b/src/lib/deep-research.ts
index 3754084f..5abf0230 100644
--- a/src/lib/deep-research.ts
+++ b/src/lib/deep-research.ts
@@ -75,8 +75,10 @@ export function addResearchTaskDiscriminator(fileName: string, taskId: string):
}
export function researchPageIdFromPath(filePath: string): string {
- const fileName = filePath.split(/[\\/]/).pop() || filePath
- return fileName.replace(/\.md$/i, "")
+ const normalized = filePath.replace(/\\/g, "/")
+ const wikiIndex = normalized.lastIndexOf("/wiki/")
+ const relative = wikiIndex >= 0 ? normalized.slice(wikiIndex + 6) : normalized
+ return relative.replace(/^wiki\//, "").replace(/\.md$/i, "")
}
interface ResearchSourceDeps {
@@ -532,8 +534,14 @@ async function executeResearch(
const embeddingConfig = useWikiStore.getState().embeddingConfig
if (embeddingConfig.enabled && embeddingConfig.model) {
try {
- const { embedPage } = await import("@/lib/embedding")
- await embedPage(pp, researchPageIdFromPath(filePath), `Research: ${topic}`, pageContent, embeddingConfig)
+ const { embedPage, wikiPageIdFromPath } = await import("@/lib/embedding")
+ await embedPage(
+ pp,
+ wikiPageIdFromPath(pp, filePath),
+ `Research: ${topic}`,
+ pageContent,
+ embeddingConfig,
+ )
} catch (err) {
console.warn("[DeepResearch] failed to index generated query page:", err)
}
diff --git a/src/lib/embedding.test.ts b/src/lib/embedding.test.ts
index 2b7a8d45..5979f191 100644
--- a/src/lib/embedding.test.ts
+++ b/src/lib/embedding.test.ts
@@ -25,6 +25,7 @@ const mockHttpFetch = vi.fn<(url: string, opts?: RequestInit) => Promise<Respons
vi.mock("@/commands/fs", () => ({
readFile: vi.fn(),
listDirectory: vi.fn(),
+ fileExists: vi.fn().mockResolvedValue(false),
}))
import {
@@ -39,6 +40,7 @@ import {
removePageEmbedding,
resetEmbeddingOptimizeAccountingForTests,
extractEmbeddingTitle,
+ wikiPageIdFromPath,
type PageSearchResult,
} from "./embedding"
@@ -49,6 +51,22 @@ const cfg = {
model: "test-embed",
}
+describe("wikiPageIdFromPath", () => {
+ it("uses the wiki-relative path and normalizes Windows separators", () => {
+ expect(wikiPageIdFromPath("/project", "/project/wiki/entities/topic.md"))
+ .toBe("entities/topic")
+ expect(wikiPageIdFromPath("C:\\project", "C:\\project\\wiki\\sources\\topic.MD"))
+ .toBe("sources/topic")
+ expect(wikiPageIdFromPath("/project/", "wiki/entities/topic.md"))
+ .toBe("entities/topic")
+ })
+
+ it("keeps plain legacy slugs usable", () => {
+ expect(wikiPageIdFromPath("/project", "topic")).toBe("topic")
+ expect(wikiPageIdFromPath("/project", "/other/wiki/topic.md")).toBe("")
+ })
+})
+
/** Build an embedding-shaped JSON Response. */
function okResponse(embedding: number[]): Response {
return new Response(JSON.stringify({ data: [{ embedding }] }), {
@@ -1596,7 +1614,7 @@ describe("embedAllPages", () => {
const upsertCalls = mockInvoke.mock.calls.filter((c) => c[0] === "vector_upsert_chunks")
expect(upsertCalls).toHaveLength(2)
const pageIds = upsertCalls.map((c) => (c[1] as { pageId: string }).pageId).sort()
- expect(pageIds).toEqual(["attention", "rope"])
+ expect(pageIds).toEqual(["rope", "sub/attention"])
})
it("clears the chunk table before a forced rebuild", async () => {
diff --git a/src/lib/embedding.ts b/src/lib/embedding.ts
index 0a384b05..91a1bc36 100644
--- a/src/lib/embedding.ts
+++ b/src/lib/embedding.ts
@@ -49,6 +49,8 @@ export function getLastEmbeddingError(): string | null {
export function resetEmbeddingOptimizeAccountingForTests(): void {
incrementalOptimizeCounts.clear()
+ attemptedLegacyVectorCleanup.clear()
+ legacyStemOwnerCounts.clear()
embeddingFailureVersion = 0
lastEmbeddingError = null
}
@@ -219,6 +221,57 @@ async function vectorDeletePage(projectPath: string, pageId: string): Promise<vo
})
}
+const attemptedLegacyVectorCleanup = new Set<string>()
+const legacyStemOwnerCounts = new Map<string, Promise<Map<string, number>>>()
+
+function projectStemOwnerCounts(projectPath: string): Promise<Map<string, number>> {
+ const pp = normalizePath(projectPath)
+ const cached = legacyStemOwnerCounts.get(pp)
+ if (cached) return cached
+ const pending = Promise.resolve(listDirectory(`${pp}/wiki`)).then((tree) => {
+ const counts = new Map<string, number>()
+ const walk = (nodes: FileNode[]) => {
+ for (const node of nodes) {
+ if (node.is_dir && node.children) walk(node.children)
+ else if (!node.is_dir && /\.md$/i.test(node.name)) {
+ const stem = node.name.replace(/\.md$/i, "").toLowerCase()
+ counts.set(stem, (counts.get(stem) ?? 0) + 1)
+ }
+ }
+ }
+ walk(tree)
+ return counts
+ }).catch(() => new Map<string, number>())
+ legacyStemOwnerCounts.set(pp, pending)
+ return pending
+}
+
+async function removeLegacyVectorIdIfSafe(projectPath: string, pageId: string): Promise<void> {
+ if (!pageId.includes("/")) return
+ const legacyId = pageId.split("/").pop() ?? ""
+ if (!legacyId) return
+ const cleanupKey = `${normalizePath(projectPath)}\0${legacyId}`
+ if (attemptedLegacyVectorCleanup.has(cleanupKey)) return
+ // A legacy basename can only be removed when exactly one wiki page owns the
+ // stem. With siblings, the row may be the only vectors for another page.
+ const ownerCounts = await projectStemOwnerCounts(projectPath)
+ if ((ownerCounts.get(legacyId.toLowerCase()) ?? 0) !== 1) return
+ attemptedLegacyVectorCleanup.add(cleanupKey)
+ await vectorDeletePage(projectPath, legacyId).catch((error) => {
+ attemptedLegacyVectorCleanup.delete(cleanupKey)
+ console.warn(`[Embedding] Failed to remove legacy vector id "${legacyId}":`, error)
+ })
+}
+
+async function vectorUpsertPageChunks(
+ projectPath: string,
+ pageId: string,
+ rows: ChunkUpsertInput[],
+): Promise<void> {
+ await vectorUpsertChunks(projectPath, pageId, rows)
+ await removeLegacyVectorIdIfSafe(projectPath, pageId)
+}
+
async function vectorCountChunks(projectPath: string): Promise<number> {
return await invoke("vector_count_chunks", {
projectPath: normalizePath(projectPath),
@@ -388,6 +441,26 @@ async function preparePageEmbeddingRows(
// ── Public API: embedPage / embedAllPages / searchByEmbedding ────────────
+/**
+ * Stable vector identity for a wiki page. Directory-qualified ids prevent
+ * pages such as `sources/topic.md` and `entities/topic.md` from replacing
+ * each other's chunks. Plain slugs remain valid for compatibility with
+ * callers that do not have a page path.
+ */
+export function wikiPageIdFromPath(projectPath: string, pagePath: string): string {
+ const pp = normalizePath(projectPath).replace(/\/$/, "")
+ let normalized = normalizePath(pagePath)
+ const absoluteWikiPrefix = `${pp}/wiki/`
+ if (normalized.startsWith(absoluteWikiPrefix)) {
+ normalized = normalized.slice(absoluteWikiPrefix.length)
+ } else if (normalized.startsWith("wiki/")) {
+ normalized = normalized.slice("wiki/".length)
+ } else if (normalized.includes("/")) {
+ return ""
+ }
+ return normalized.replace(/^\/+/, "").replace(/\.md$/i, "")
+}
+
/**
* Embed a wiki page: chunk → per-chunk embed → replace the page's
* vectors in LanceDB in one batch. Every transient failure leaves the
@@ -415,7 +488,7 @@ export async function embedPage(
return false
}
- await vectorUpsertChunks(projectPath, pageId, prepared.page.rows)
+ await vectorUpsertPageChunks(projectPath, pageId, prepared.page.rows)
if (!options?.deferOptimization) {
await noteIncrementalVectorWrite(projectPath)
}
@@ -532,10 +605,10 @@ export async function embedAllPages(
for (const node of nodes) {
if (node.is_dir && node.children) {
walk(node.children)
- } else if (!node.is_dir && node.name.endsWith(".md")) {
- const id = node.name.replace(/\.md$/, "")
- if (!["index", "log", "overview", "purpose", "schema"].includes(id)) {
- mdFiles.push({ id, path: node.path })
+ } else if (!node.is_dir && /\.md$/i.test(node.name)) {
+ const stem = node.name.replace(/\.md$/i, "")
+ if (!["index", "log", "overview", "purpose", "schema"].includes(stem)) {
+ mdFiles.push({ id: wikiPageIdFromPath(pp, node.path), path: node.path })
}
}
}
@@ -605,7 +678,7 @@ export async function embedAllPages(
let updated = 0
for (const page of preparedPages) {
try {
- await vectorUpsertChunks(pp, page.pageId, page.rows)
+ await vectorUpsertPageChunks(pp, page.pageId, page.rows)
updated++
} catch (err) {
failures.push(
@@ -634,6 +707,7 @@ export async function embedAllPages(
let written = 0
for (const page of preparedPages) {
try {
+ // The table was just cleared, so no legacy basename rows remain.
await vectorUpsertChunks(pp, page.pageId, page.rows)
} catch (err) {
const reason = err instanceof Error ? err.message : String(err)
@@ -668,7 +742,7 @@ export async function embedAllPages(
const title = extractEmbeddingTitle(content, file.id)
const prepared = await preparePageEmbeddingRows(file.id, title, content, cfg, scheduleEmbedding)
if (prepared.status === "ready") {
- await scheduleVectorWrite(() => vectorUpsertChunks(pp, file.id, prepared.page.rows))
+ await scheduleVectorWrite(() => vectorUpsertPageChunks(pp, file.id, prepared.page.rows))
indexed++
}
} catch {
@@ -777,6 +851,7 @@ export async function removePageEmbedding(
): Promise<void> {
try {
await vectorDeletePage(projectPath, pageId)
+ await removeLegacyVectorIdIfSafe(projectPath, pageId)
} catch {
// non-critical
}
diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts
index 6a3a8879..360b5af9 100644
--- a/src/lib/ingest.ts
+++ b/src/lib/ingest.ts
@@ -1481,10 +1481,11 @@ async function autoIngestImpl(
const embCfg = useWikiStore.getState().embeddingConfig
if (embCfg.enabled && embCfg.model && writtenPaths.length > 0) {
try {
- const { embedPage } = await import("@/lib/embedding")
+ const { embedPage, wikiPageIdFromPath } = await import("@/lib/embedding")
for (const wpath of writtenPaths) {
- const pageId = wpath.split("/").pop()?.replace(/\.md$/, "") ?? ""
- if (!pageId || ["index", "log", "overview"].includes(pageId)) continue
+ const pageId = wikiPageIdFromPath(pp, wpath)
+ const pageStem = wpath.split("/").pop()?.replace(/\.md$/, "") ?? ""
+ if (!pageId || ["index", "log", "overview"].includes(pageStem)) continue
try {
const content = await readFile(`${pp}/${wpath}`)
const fmTitle = parseFrontmatter(content).frontmatter?.title
@@ -3283,8 +3284,14 @@ async function reembedSourceSummary(
const content = await readFile(sourceSummaryFullPath)
const fmTitle = parseFrontmatter(content).frontmatter?.title
const title = typeof fmTitle === "string" && fmTitle.trim() ? fmTitle.trim() : sourceIdentity
- const { embedPage } = await import("@/lib/embedding")
- await embedPage(pp, sourceSummarySlug, title, content, embCfg)
+ const { embedPage, wikiPageIdFromPath } = await import("@/lib/embedding")
+ await embedPage(
+ pp,
+ wikiPageIdFromPath(pp, sourceSummaryFullPath),
+ title,
+ content,
+ embCfg,
+ )
console.log(`[ingest:caption] re-embedded ${sourceSummarySlug} with captioned alt text`)
} catch (err) {
console.warn(
diff --git a/src/lib/project-file-sync.test.ts b/src/lib/project-file-sync.test.ts
index 49b9196f..12e02d6a 100644
--- a/src/lib/project-file-sync.test.ts
+++ b/src/lib/project-file-sync.test.ts
@@ -107,6 +107,14 @@ const mocks = vi.hoisted(() => {
removeFromIngestCache: vi.fn(async () => undefined),
moveIngestCacheEntry: vi.fn(async () => undefined),
removePageEmbedding: vi.fn(async () => undefined),
+ wikiPageIdFromPath: vi.fn((projectPath: string, pagePath: string) => {
+ const normalizedProject = projectPath.replace(/\\/g, "/").replace(/\/$/, "")
+ const normalizedPage = pagePath.replace(/\\/g, "/")
+ const relative = normalizedPage.startsWith(`${normalizedProject}/`)
+ ? normalizedPage.slice(normalizedProject.length + 1)
+ : normalizedPage
+ return relative.replace(/^wiki\//i, "").replace(/\.md$/i, "")
+ }),
cascadeDeleteWikiPagesWithRefs: vi.fn(async () => ({
deletedPaths: [] as string[],
rewrittenFiles: 0,
@@ -153,6 +161,7 @@ vi.mock("@/lib/ingest-cache", () => ({
vi.mock("@/lib/embedding", () => ({
removePageEmbedding: mocks.removePageEmbedding,
+ wikiPageIdFromPath: mocks.wikiPageIdFromPath,
}))
vi.mock("@/lib/wiki-page-delete", () => ({
diff --git a/src/lib/project-file-sync.ts b/src/lib/project-file-sync.ts
index 5933b66c..ed7049e2 100644
--- a/src/lib/project-file-sync.ts
+++ b/src/lib/project-file-sync.ts
@@ -9,7 +9,8 @@ import {
} from "@/commands/file-sync"
import { useFileSyncStore } from "@/stores/file-sync-store"
import { useWikiStore } from "@/stores/wiki-store"
-import { getFileStem, normalizePath } from "@/lib/path-utils"
+import { normalizePath } from "@/lib/path-utils"
+import { wikiPageIdFromPath } from "@/lib/embedding"
import type { WikiProject } from "@/types/wiki"
import type { SourceWatchConfig } from "@/stores/wiki-store"
import type { FileChangeTask } from "@/commands/file-sync"
@@ -395,20 +396,24 @@ async function cleanupDeletedFiles(project: WikiProject, tasks: FileChangeTask[]
const rawSources = deleted.filter(isRawSourcePathForCascade)
const wikiPages = deleted.filter(isWikiPageForCascade)
- let deletedWikiSlugs = new Set<string>()
+ let deletedWikiIds = new Set<string>()
if (rawSources.length > 0) {
try {
const result = await deleteSourceFiles(project.path, rawSources, {
fileAlreadyDeleted: true,
logReason: rawSources.length === 1 ? "external delete" : "external batch delete",
})
- deletedWikiSlugs = new Set(result.deletedWikiPaths.map((path) => getFileStem(path)))
+ deletedWikiIds = new Set(
+ result.deletedWikiPaths.map((path) => wikiPageIdFromPath(project.path, path)),
+ )
} catch (err) {
console.error("[file-sync] failed to clean deleted raw sources:", err)
}
}
- const wikiPagesToClean = wikiPages.filter((path) => !deletedWikiSlugs.has(getFileStem(path)))
+ const wikiPagesToClean = wikiPages.filter(
+ (path) => !deletedWikiIds.has(wikiPageIdFromPath(project.path, path)),
+ )
if (wikiPagesToClean.length > 0) {
try {
await cleanupDeletedWikiPages(project.path, wikiPagesToClean)
diff --git a/src/lib/source-lifecycle.ts b/src/lib/source-lifecycle.ts
index 408c4aae..a51b1ab8 100644
--- a/src/lib/source-lifecycle.ts
+++ b/src/lib/source-lifecycle.ts
@@ -26,7 +26,7 @@ import {
writeSources,
} from "@/lib/sources-merge"
import { moveIngestCacheEntry, removeFromIngestCache } from "@/lib/ingest-cache"
-import { removePageEmbedding } from "@/lib/embedding"
+import { removePageEmbedding, wikiPageIdFromPath } from "@/lib/embedding"
import {
buildDeletedKeys,
cleanIndexListing,
@@ -606,17 +606,20 @@ export async function cleanupDeletedWikiPages(
): Promise<void> {
const pp = normalizePath(projectPath)
const deletedInfos = relativePaths
- .map((path) => ({ slug: getFileStem(path), title: "" }))
+ .map((path) => ({ path, slug: getFileStem(path), title: "" }))
.filter((info) => info.slug.length > 0 && !info.slug.startsWith("."))
if (deletedInfos.length === 0) return
for (const info of deletedInfos) {
- await removePageEmbedding(pp, info.slug)
- try {
- await deleteFile(`${pp}/wiki/media/${info.slug}`)
- } catch {
- // only source-summary pages usually own media; absence is normal
+ await removePageEmbedding(pp, wikiPageIdFromPath(pp, info.path))
+ const normalizedPath = normalizePath(info.path)
+ if (normalizedPath.includes("/wiki/sources/") || normalizedPath.startsWith("wiki/sources/")) {
+ try {
+ await deleteFile(`${pp}/wiki/media/${info.slug}`)
+ } catch {
+ // Source pages may have no extracted media directory.
+ }
}
}
diff --git a/src/lib/wiki-page-delete.test.ts b/src/lib/wiki-page-delete.test.ts
index 408516a2..a7735371 100644
--- a/src/lib/wiki-page-delete.test.ts
+++ b/src/lib/wiki-page-delete.test.ts
@@ -19,7 +19,8 @@ vi.mock("@/commands/fs", () => ({
listDirectory: (path: string) => mockListDirectory(path),
}))
-vi.mock("@/lib/embedding", () => ({
+vi.mock("@/lib/embedding", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@/lib/embedding")>()),
removePageEmbedding: (projectPath: string, slug: string) =>
mockRemovePageEmbedding(projectPath, slug),
}))
@@ -47,7 +48,7 @@ describe("cascadeDeleteWikiPage", () => {
expect(mockDeleteFile).toHaveBeenCalledWith("/proj/wiki/concepts/rope.md")
expect(mockRemovePageEmbedding).toHaveBeenCalledTimes(1)
- expect(mockRemovePageEmbedding).toHaveBeenCalledWith("/proj", "rope")
+ expect(mockRemovePageEmbedding).toHaveBeenCalledWith("/proj", "concepts/rope")
})
it("calls deleteFile BEFORE removePageEmbedding (file is the source of truth)", async () => {
@@ -95,9 +96,9 @@ describe("cascadeDeleteWikiPage", () => {
expect(mockDeleteFile).toHaveBeenCalled()
})
- it("derives slug from the path's basename, ignoring directory segments", async () => {
+ it("qualifies the page id with its wiki-relative directory", async () => {
await cascadeDeleteWikiPage("/proj", "/proj/wiki/concepts/some-deep/nested/page.md")
- expect(mockRemovePageEmbedding).toHaveBeenCalledWith("/proj", "page")
+ expect(mockRemovePageEmbedding).toHaveBeenCalledWith("/proj", "concepts/some-deep/nested/page")
})
it("handles Windows backslash paths (project path normalization happens elsewhere)", async () => {
@@ -107,7 +108,7 @@ describe("cascadeDeleteWikiPage", () => {
await cascadeDeleteWikiPage("C:/proj", "C:\\proj\\wiki\\entities\\transformer.md")
expect(mockDeleteFile).toHaveBeenCalledWith("C:\\proj\\wiki\\entities\\transformer.md")
- expect(mockRemovePageEmbedding).toHaveBeenCalledWith("C:/proj", "transformer")
+ expect(mockRemovePageEmbedding).toHaveBeenCalledWith("C:/proj", "entities/transformer")
})
it("preserves dotted page names (e.g. foo.bar.md) in the slug", async () => {
@@ -115,7 +116,7 @@ describe("cascadeDeleteWikiPage", () => {
// Pin it: a regression that strips ALL dots would turn this slug
// into "foo" and orphan the LanceDB chunks for "foo.bar".
await cascadeDeleteWikiPage("/proj", "/proj/wiki/concepts/foo.bar.md")
- expect(mockRemovePageEmbedding).toHaveBeenCalledWith("/proj", "foo.bar")
+ expect(mockRemovePageEmbedding).toHaveBeenCalledWith("/proj", "concepts/foo.bar")
})
it("skips removePageEmbedding when slug derivation yields empty (defensive)", async () => {
@@ -144,7 +145,7 @@ describe("cascadeDeleteWikiPage", () => {
expect(mockDeleteFile).toHaveBeenCalledTimes(2)
expect(mockDeleteFile).toHaveBeenNthCalledWith(1, "/proj/wiki/sources/rope-paper.md")
expect(mockDeleteFile).toHaveBeenNthCalledWith(2, "/proj/wiki/media/rope-paper")
- expect(mockRemovePageEmbedding).toHaveBeenCalledWith("/proj", "rope-paper")
+ expect(mockRemovePageEmbedding).toHaveBeenCalledWith("/proj", "sources/rope-paper")
})
it("does NOT cascade media when deleting a non-source page (concept / entity / queries)", async () => {
@@ -179,7 +180,7 @@ describe("cascadeDeleteWikiPage", () => {
// Both attempts happened.
expect(mockDeleteFile).toHaveBeenCalledTimes(2)
// Embedding cascade still ran in between.
- expect(mockRemovePageEmbedding).toHaveBeenCalledWith("/proj", "text-only-source")
+ expect(mockRemovePageEmbedding).toHaveBeenCalledWith("/proj", "sources/text-only-source")
})
it("handles Windows backslash paths in the source-page detection", async () => {
@@ -259,7 +260,7 @@ describe("cascadeDeleteWikiPagesWithRefs", () => {
const result = await cascadeDeleteWikiPagesWithRefs(PROJECT, [target])
expect(result.deletedPaths).toEqual([target])
expect(mockDeleteFile).toHaveBeenCalledWith(target)
- expect(mockRemovePageEmbedding).toHaveBeenCalledWith(PROJECT, "alice-chen")
+ expect(mockRemovePageEmbedding).toHaveBeenCalledWith(PROJECT, "entities/alice-chen")
})
it("strips [[deleted]] body wikilinks from sibling pages", async () => {
diff --git a/src/lib/wiki-page-delete.ts b/src/lib/wiki-page-delete.ts
index a558fc14..98f829ed 100644
--- a/src/lib/wiki-page-delete.ts
+++ b/src/lib/wiki-page-delete.ts
@@ -21,7 +21,7 @@
*/
import { deleteFile, listDirectory, readFile, writeFile } from "@/commands/fs"
import { getFileStem, normalizePath } from "@/lib/path-utils"
-import { removePageEmbedding } from "@/lib/embedding"
+import { removePageEmbedding, wikiPageIdFromPath } from "@/lib/embedding"
import {
buildDeletedKeys,
cleanIndexListing,
@@ -61,10 +61,9 @@ function isSourcePage(pagePath: string): boolean {
* cascade to the right LanceDB instance, and to locate the media
* directory).
*
- * `pagePath` may be absolute or relative; only its basename is used
- * for the page-id lookup, so callers don't need to normalize before
- * calling. The disk delete uses the path verbatim — pass an
- * absolute path if your caller has one (most do).
+ * `pagePath` may be absolute or project-relative. Its wiki-relative path is
+ * used as the vector identity so same-named pages in different schema
+ * directories remain independent.
*/
export async function cascadeDeleteWikiPage(
projectPath: string,
@@ -72,8 +71,9 @@ export async function cascadeDeleteWikiPage(
): Promise<void> {
await deleteFile(pagePath)
const slug = getFileStem(pagePath)
- if (slug.length > 0) {
- await removePageEmbedding(projectPath, slug)
+ const pageId = wikiPageIdFromPath(projectPath, pagePath)
+ if (pageId.length > 0) {
+ await removePageEmbedding(projectPath, pageId)
}
// Media cascade: source-summary deletion → drop the source's