World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Retry an interrupted full-text index rebuild instead of keeping its first documents (#1256)
Maya Kid800d317cbc78
diff --git a/crates/utopia-search/src/lib.rs b/crates/utopia-search/src/lib.rsindex 95429d7f..4fe45b51 100644--- a/crates/utopia-search/src/lib.rs+++ b/crates/utopia-search/src/lib.rs@@ -102,6 +102,16 @@ impl SearchIndex {Ok(())}+ /// Discard an incomplete rebuild while retaining the index's writer lock.+ /// Removing its directory first could erase another process's live index.+ pub fn clear(&self) -> anyhow::Result<()> {+ let mut writer = self.writer.lock().expect("writer 锁中毒");+ writer.delete_all_documents()?;+ writer.commit()?;+ self.reader.reload()?;+ Ok(())+ }+pub fn delete_document(&self, document_id: &str) -> anyhow::Result<()> {let mut writer = self.writer.lock().expect("writer 锁中毒");writer.delete_term(Term::from_field_text(self.f_document_id, document_id));diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex f8f14eb9..0e30b669 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -35,6 +35,7 @@ mod rdf;mod readers;mod retrieval;mod rss_full_content;+mod search_recovery;mod state;mod time_resolution;mod time_text;@@ -139,7 +140,7 @@ async fn main() -> anyhow::Result<()> {}let index_dir = std::path::Path::new(&cfg.data_dir).join("index");- let search = Arc::new(SearchIndex::open(&index_dir)?);+ let search = Arc::new(search_recovery::open(&index_dir)?);tracing::info!("全文索引就绪: {}", index_dir.display());// **索引落空就自己重建,不给按钮。**@@ -150,7 +151,7 @@ async fn main() -> anyhow::Result<()> {//// 判据是「库里有分块而索引空着」,不是「数目对不对得上」:后者在正常运行中// 也会短暂不等(一篇文档正在索引),拿它当判据会让每次启动都重建一遍。- reindex_if_empty(&pool, &search).await;+ reindex_if_empty(&pool, &search, &index_dir).await;// JWT 密钥:环境变量优先(轮换、多实例显式对齐走这条),否则用库里那条;// 库里也没有就现生成一条存进去。生成放在这里而不是 store 里,是因为@@ -340,21 +341,10 @@ async fn main() -> anyhow::Result<()> {////// 失败只告警不阻断启动:检索退化成「暂时搜不到」,而整个服务起不来是更坏的/// 结果。日志里那一行说清了发生什么,下次重启还会再试。-async fn reindex_if_empty(pool: &sqlx::PgPool, search: &SearchIndex) {+async fn reindex_if_empty(pool: &sqlx::PgPool, search: &SearchIndex, index_dir: &std::path::Path) {if !search.is_empty() {return;}- let total = match utopia_store::documents::live_chunk_count(pool).await {- Ok(n) => n,- Err(e) => {- tracing::warn!(error = %e, "数不出分块数,跳过索引重建");- return;- }- };- if total == 0 {- return;- }- tracing::info!(chunks = total, "全文索引是空的,从库里重建");let rows = match utopia_store::documents::all_chunks_for_index(pool).await {Ok(r) => r,Err(e) => {@@ -362,30 +352,17 @@ async fn reindex_if_empty(pool: &sqlx::PgPool, search: &SearchIndex) {return;}};- // 按文档攒批再写:`reindex_document` 一次替换一篇文档的全部分块,- // 逐条调会把前一条刚写的删掉- let mut current: Option<(uuid::Uuid, uuid::Uuid)> = None;- let mut batch: Vec<(String, String)> = Vec::new();- let mut done = 0usize;- let flush = |key: Option<(uuid::Uuid, uuid::Uuid)>, batch: &mut Vec<(String, String)>| {- if let Some((kb, doc)) = key {- if let Err(e) = search.reindex_document(&kb.to_string(), &doc.to_string(), batch) {- tracing::warn!(error = %e, document = %doc, "重建索引时有一篇没写进去");- }- }- batch.clear();- };- for (kb_id, doc_id, chunk_id, text) in rows {- if current != Some((kb_id, doc_id)) {- flush(current, &mut batch);- current = Some((kb_id, doc_id));- }- batch.push((chunk_id.to_string(), text));- done += 1;+ let total = rows.len();+ if total > 0 {+ tracing::info!(chunks = total, "全文索引是空的,从库里重建");+ }+ // Even an empty database completes an interrupted rebuild: otherwise the+ // marker would make every subsequent startup discard an empty generation.+ match search_recovery::rebuild(index_dir, search, rows) {+ Ok(()) if total > 0 => tracing::info!(chunks = total, "全文索引重建完成"),+ Ok(()) => {}+ Err(e) => tracing::warn!(error = %e, "全文索引未重建完,下次启动重试"),}- // `reindex_document` 自己 commit,所以这里不必再提交一次- flush(current, &mut batch);- tracing::info!(chunks = done, "全文索引重建完成");}////// 任务分发:新任务类型在这里注册。diff --git a/crates/utopia-server/src/search_recovery.rs b/crates/utopia-server/src/search_recovery.rsnew file mode 100644index 00000000..0316457e--- /dev/null+++ b/crates/utopia-server/src/search_recovery.rs@@ -0,0 +1,169 @@+//! A document commit is not a completed rebuild. Keep the marker outside the+//! index so recovery state does not depend on the generation being rebuilt.++use std::fs::{self, File};+use std::path::{Path, PathBuf};+use utopia_search::SearchIndex;+use uuid::Uuid;++type Row = (Uuid, Uuid, Uuid, String);++fn marker(index_dir: &Path) -> PathBuf {+ index_dir.with_extension("rebuilding")+}++fn sync_parent(path: &Path) -> anyhow::Result<()> {+ // The marker must survive a crash after the first document commits, including+ // its directory entry. Opening directories for syncing is Unix-specific.+ #[cfg(unix)]+ File::open(+ path.parent()+ .filter(|p| !p.as_os_str().is_empty())+ .unwrap_or(Path::new(".")),+ )?+ .sync_all()?;+ #[cfg(not(unix))]+ let _ = path;+ Ok(())+}++pub fn open(index_dir: &Path) -> anyhow::Result<SearchIndex> {+ // Opening first acquires Tantivy's writer lock. A second process must fail+ // here, before it can reset an index the first process is still rebuilding.+ let search = SearchIndex::open(index_dir)?;+ if marker(index_dir).try_exists()? {+ tracing::info!("全文索引重建曾中断,从库里重新开始");+ // Retrying in place would retain documents deleted from the database+ // between attempts. Keep the marker until the fresh run finishes.+ search.clear()?;+ }+ Ok(search)+}++pub fn rebuild(index_dir: &Path, search: &SearchIndex, rows: Vec<Row>) -> anyhow::Result<()> {+ rebuild_with(index_dir, rows, |kb, doc, batch| {+ search.reindex_document(kb, doc, batch)+ })+}++fn rebuild_with(+ index_dir: &Path,+ rows: Vec<Row>,+ mut write: impl FnMut(&str, &str, &[(String, String)]) -> anyhow::Result<()>,+) -> anyhow::Result<()> {+ let marker = marker(index_dir);+ File::create(&marker)?.sync_all()?;+ sync_parent(&marker)?;++ // The store returns document_id/seq order. Each call replaces a whole+ // document; sending its chunks individually would delete preceding chunks.+ let mut rows = rows.into_iter().peekable();+ while let Some((kb, doc, chunk, text)) = rows.next() {+ let mut batch = vec![(chunk.to_string(), text)];+ while rows.peek().is_some_and(|(_, next, _, _)| *next == doc) {+ let (_, _, chunk, text) = rows.next().expect("peeked row");+ batch.push((chunk.to_string(), text));+ }+ // Propagate a single-document failure: the next boot must retry even+ // when preceding documents have made the index nonempty.+ write(&kb.to_string(), &doc.to_string(), &batch)?;+ }+ fs::remove_file(&marker)?;+ sync_parent(&marker)?;+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;++ fn rows(kb: Uuid, docs: [Uuid; 2]) -> Vec<Row> {+ vec![+ (kb, docs[0], Uuid::now_v7(), "firstneedle".into()),+ (kb, docs[0], Uuid::now_v7(), "secondneedle".into()),+ (kb, docs[1], Uuid::now_v7(), "thirdneedle".into()),+ ]+ }++ #[test]+ fn a_restart_retries_an_interrupted_or_failed_rebuild() -> anyhow::Result<()> {+ for interrupted in [true, false] {+ let dir = tempfile::tempdir()?;+ let index_dir = dir.path().join("index");+ let kb = Uuid::now_v7();+ let docs = [Uuid::now_v7(), Uuid::now_v7()];+ let search = open(&index_dir)?;+ let attempt = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {+ rebuild_with(&index_dir, rows(kb, docs), |kb, doc, batch| {+ if doc == docs[1].to_string() {+ if interrupted {+ panic!("interrupted after the first document committed");+ }+ anyhow::bail!("one document could not be indexed");+ }+ search.reindex_document(kb, doc, batch)+ })+ }));+ assert!(attempt.is_err() || attempt.unwrap().is_err());+ assert_eq!(search.len(), 2, "the first document really committed");+ assert!(marker(&index_dir).exists());+ assert!(open(&index_dir).is_err(), "a second writer must be refused");+ assert_eq!(search.len(), 2, "the live writer keeps its committed data");+ assert_eq!(search.search(&kb.to_string(), "firstneedle", 10)?.len(), 1);+ drop(search);++ let search = open(&index_dir)?;+ assert!(search.is_empty(), "discard the incomplete generation");+ // The first document was deleted between attempts; none of its old+ // chunks may survive, and both chunks of the live document belong.+ let live = vec![+ (kb, docs[1], Uuid::now_v7(), "thirdneedle".into()),+ (kb, docs[1], Uuid::now_v7(), "fourthneedle".into()),+ ];+ rebuild(&index_dir, &search, live)?;+ assert_eq!(search.len(), 2);+ assert!(search+ .search(&kb.to_string(), "firstneedle", 10)?+ .is_empty());+ assert_eq!(search.search(&kb.to_string(), "thirdneedle", 10)?.len(), 1);+ assert!(!marker(&index_dir).exists());+ }+ Ok(())+ }++ #[test]+ fn a_complete_or_legacy_index_is_reused_on_restart() -> anyhow::Result<()> {+ let dir = tempfile::tempdir()?;+ let index_dir = dir.path().join("index");+ let kb = Uuid::now_v7();+ let docs = [Uuid::now_v7(), Uuid::now_v7()];+ let search = open(&index_dir)?;+ rebuild(&index_dir, &search, rows(kb, docs))?;+ assert!(!marker(&index_dir).exists());+ drop(search);++ let search = open(&index_dir)?;+ assert_eq!(search.len(), 3);+ // A legacy index has no marker either: ordinary writes and the next+ // open must not erase it or turn a normal startup into a rebuild.+ search.reindex_document(+ &kb.to_string(),+ &docs[0].to_string(),+ &[(Uuid::now_v7().to_string(), "legacyneedle".into())],+ )?;+ drop(search);+ let search = open(&index_dir)?;+ assert_eq!(search.len(), 2);+ assert_eq!(search.search(&kb.to_string(), "legacyneedle", 10)?.len(), 1);++ // The database may become empty between attempts. Completing that+ // recovery must clear its marker instead of rebuilding on every boot.+ File::create(marker(&index_dir))?.sync_all()?;+ drop(search);+ let search = open(&index_dir)?;+ assert!(search.is_empty());+ rebuild(&index_dir, &search, Vec::new())?;+ assert!(!marker(&index_dir).exists());+ Ok(())+ }+}