Skip to content

World's first open-source enterprise world model.

Retry an interrupted full-text index rebuild instead of keeping its first documents (#1256)

Maya Kid800d317cbc78
diff --git a/crates/utopia-search/src/lib.rs b/crates/utopia-search/src/lib.rs
index 95429d7f..4fe45b51 100644
--- a/crates/utopia-search/src/lib.rs
+++ b/crates/utopia-search/src/lib.rs
@@ -102,6 +102,16 @@ impl SearchIndex {
Ok(())
}
+ /// Discard an incomplete rebuild while retaining the index's writer lock.
+ /// Removing its directory first could erase another process's live index.
+ pub fn clear(&self) -> anyhow::Result<()> {
+ let mut writer = self.writer.lock().expect("writer 锁中毒");
+ writer.delete_all_documents()?;
+ writer.commit()?;
+ self.reader.reload()?;
+ Ok(())
+ }
+
pub fn delete_document(&self, document_id: &str) -> anyhow::Result<()> {
let mut writer = self.writer.lock().expect("writer 锁中毒");
writer.delete_term(Term::from_field_text(self.f_document_id, document_id));
diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs
index f8f14eb9..0e30b669 100644
--- a/crates/utopia-server/src/main.rs
+++ b/crates/utopia-server/src/main.rs
@@ -35,6 +35,7 @@ mod rdf;
mod readers;
mod retrieval;
mod rss_full_content;
+mod search_recovery;
mod state;
mod time_resolution;
mod time_text;
@@ -139,7 +140,7 @@ async fn main() -> anyhow::Result<()> {
}
let index_dir = std::path::Path::new(&cfg.data_dir).join("index");
- let search = Arc::new(SearchIndex::open(&index_dir)?);
+ let search = Arc::new(search_recovery::open(&index_dir)?);
tracing::info!("全文索引就绪: {}", index_dir.display());
// **索引落空就自己重建,不给按钮。**
@@ -150,7 +151,7 @@ async fn main() -> anyhow::Result<()> {
//
// 判据是「库里有分块而索引空着」,不是「数目对不对得上」:后者在正常运行中
// 也会短暂不等(一篇文档正在索引),拿它当判据会让每次启动都重建一遍。
- reindex_if_empty(&pool, &search).await;
+ reindex_if_empty(&pool, &search, &index_dir).await;
// JWT 密钥:环境变量优先(轮换、多实例显式对齐走这条),否则用库里那条;
// 库里也没有就现生成一条存进去。生成放在这里而不是 store 里,是因为
@@ -340,21 +341,10 @@ async fn main() -> anyhow::Result<()> {
///
/// 失败只告警不阻断启动:检索退化成「暂时搜不到」,而整个服务起不来是更坏的
/// 结果。日志里那一行说清了发生什么,下次重启还会再试。
-async fn reindex_if_empty(pool: &sqlx::PgPool, search: &SearchIndex) {
+async fn reindex_if_empty(pool: &sqlx::PgPool, search: &SearchIndex, index_dir: &std::path::Path) {
if !search.is_empty() {
return;
}
- let total = match utopia_store::documents::live_chunk_count(pool).await {
- Ok(n) => n,
- Err(e) => {
- tracing::warn!(error = %e, "数不出分块数,跳过索引重建");
- return;
- }
- };
- if total == 0 {
- return;
- }
- tracing::info!(chunks = total, "全文索引是空的,从库里重建");
let rows = match utopia_store::documents::all_chunks_for_index(pool).await {
Ok(r) => r,
Err(e) => {
@@ -362,30 +352,17 @@ async fn reindex_if_empty(pool: &sqlx::PgPool, search: &SearchIndex) {
return;
}
};
- // 按文档攒批再写:`reindex_document` 一次替换一篇文档的全部分块,
- // 逐条调会把前一条刚写的删掉
- let mut current: Option<(uuid::Uuid, uuid::Uuid)> = None;
- let mut batch: Vec<(String, String)> = Vec::new();
- let mut done = 0usize;
- let flush = |key: Option<(uuid::Uuid, uuid::Uuid)>, batch: &mut Vec<(String, String)>| {
- if let Some((kb, doc)) = key {
- if let Err(e) = search.reindex_document(&kb.to_string(), &doc.to_string(), batch) {
- tracing::warn!(error = %e, document = %doc, "重建索引时有一篇没写进去");
- }
- }
- batch.clear();
- };
- for (kb_id, doc_id, chunk_id, text) in rows {
- if current != Some((kb_id, doc_id)) {
- flush(current, &mut batch);
- current = Some((kb_id, doc_id));
- }
- batch.push((chunk_id.to_string(), text));
- done += 1;
+ let total = rows.len();
+ if total > 0 {
+ tracing::info!(chunks = total, "全文索引是空的,从库里重建");
+ }
+ // Even an empty database completes an interrupted rebuild: otherwise the
+ // marker would make every subsequent startup discard an empty generation.
+ match search_recovery::rebuild(index_dir, search, rows) {
+ Ok(()) if total > 0 => tracing::info!(chunks = total, "全文索引重建完成"),
+ Ok(()) => {}
+ Err(e) => tracing::warn!(error = %e, "全文索引未重建完,下次启动重试"),
}
- // `reindex_document` 自己 commit,所以这里不必再提交一次
- flush(current, &mut batch);
- tracing::info!(chunks = done, "全文索引重建完成");
}
///
/// 任务分发:新任务类型在这里注册。
diff --git a/crates/utopia-server/src/search_recovery.rs b/crates/utopia-server/src/search_recovery.rs
new file mode 100644
index 00000000..0316457e
--- /dev/null
+++ b/crates/utopia-server/src/search_recovery.rs
@@ -0,0 +1,169 @@
+//! A document commit is not a completed rebuild. Keep the marker outside the
+//! index so recovery state does not depend on the generation being rebuilt.
+
+use std::fs::{self, File};
+use std::path::{Path, PathBuf};
+use utopia_search::SearchIndex;
+use uuid::Uuid;
+
+type Row = (Uuid, Uuid, Uuid, String);
+
+fn marker(index_dir: &Path) -> PathBuf {
+ index_dir.with_extension("rebuilding")
+}
+
+fn sync_parent(path: &Path) -> anyhow::Result<()> {
+ // The marker must survive a crash after the first document commits, including
+ // its directory entry. Opening directories for syncing is Unix-specific.
+ #[cfg(unix)]
+ File::open(
+ path.parent()
+ .filter(|p| !p.as_os_str().is_empty())
+ .unwrap_or(Path::new(".")),
+ )?
+ .sync_all()?;
+ #[cfg(not(unix))]
+ let _ = path;
+ Ok(())
+}
+
+pub fn open(index_dir: &Path) -> anyhow::Result<SearchIndex> {
+ // Opening first acquires Tantivy's writer lock. A second process must fail
+ // here, before it can reset an index the first process is still rebuilding.
+ let search = SearchIndex::open(index_dir)?;
+ if marker(index_dir).try_exists()? {
+ tracing::info!("全文索引重建曾中断,从库里重新开始");
+ // Retrying in place would retain documents deleted from the database
+ // between attempts. Keep the marker until the fresh run finishes.
+ search.clear()?;
+ }
+ Ok(search)
+}
+
+pub fn rebuild(index_dir: &Path, search: &SearchIndex, rows: Vec<Row>) -> anyhow::Result<()> {
+ rebuild_with(index_dir, rows, |kb, doc, batch| {
+ search.reindex_document(kb, doc, batch)
+ })
+}
+
+fn rebuild_with(
+ index_dir: &Path,
+ rows: Vec<Row>,
+ mut write: impl FnMut(&str, &str, &[(String, String)]) -> anyhow::Result<()>,
+) -> anyhow::Result<()> {
+ let marker = marker(index_dir);
+ File::create(&marker)?.sync_all()?;
+ sync_parent(&marker)?;
+
+ // The store returns document_id/seq order. Each call replaces a whole
+ // document; sending its chunks individually would delete preceding chunks.
+ let mut rows = rows.into_iter().peekable();
+ while let Some((kb, doc, chunk, text)) = rows.next() {
+ let mut batch = vec![(chunk.to_string(), text)];
+ while rows.peek().is_some_and(|(_, next, _, _)| *next == doc) {
+ let (_, _, chunk, text) = rows.next().expect("peeked row");
+ batch.push((chunk.to_string(), text));
+ }
+ // Propagate a single-document failure: the next boot must retry even
+ // when preceding documents have made the index nonempty.
+ write(&kb.to_string(), &doc.to_string(), &batch)?;
+ }
+ fs::remove_file(&marker)?;
+ sync_parent(&marker)?;
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn rows(kb: Uuid, docs: [Uuid; 2]) -> Vec<Row> {
+ vec![
+ (kb, docs[0], Uuid::now_v7(), "firstneedle".into()),
+ (kb, docs[0], Uuid::now_v7(), "secondneedle".into()),
+ (kb, docs[1], Uuid::now_v7(), "thirdneedle".into()),
+ ]
+ }
+
+ #[test]
+ fn a_restart_retries_an_interrupted_or_failed_rebuild() -> anyhow::Result<()> {
+ for interrupted in [true, false] {
+ let dir = tempfile::tempdir()?;
+ let index_dir = dir.path().join("index");
+ let kb = Uuid::now_v7();
+ let docs = [Uuid::now_v7(), Uuid::now_v7()];
+ let search = open(&index_dir)?;
+ let attempt = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ rebuild_with(&index_dir, rows(kb, docs), |kb, doc, batch| {
+ if doc == docs[1].to_string() {
+ if interrupted {
+ panic!("interrupted after the first document committed");
+ }
+ anyhow::bail!("one document could not be indexed");
+ }
+ search.reindex_document(kb, doc, batch)
+ })
+ }));
+ assert!(attempt.is_err() || attempt.unwrap().is_err());
+ assert_eq!(search.len(), 2, "the first document really committed");
+ assert!(marker(&index_dir).exists());
+ assert!(open(&index_dir).is_err(), "a second writer must be refused");
+ assert_eq!(search.len(), 2, "the live writer keeps its committed data");
+ assert_eq!(search.search(&kb.to_string(), "firstneedle", 10)?.len(), 1);
+ drop(search);
+
+ let search = open(&index_dir)?;
+ assert!(search.is_empty(), "discard the incomplete generation");
+ // The first document was deleted between attempts; none of its old
+ // chunks may survive, and both chunks of the live document belong.
+ let live = vec![
+ (kb, docs[1], Uuid::now_v7(), "thirdneedle".into()),
+ (kb, docs[1], Uuid::now_v7(), "fourthneedle".into()),
+ ];
+ rebuild(&index_dir, &search, live)?;
+ assert_eq!(search.len(), 2);
+ assert!(search
+ .search(&kb.to_string(), "firstneedle", 10)?
+ .is_empty());
+ assert_eq!(search.search(&kb.to_string(), "thirdneedle", 10)?.len(), 1);
+ assert!(!marker(&index_dir).exists());
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn a_complete_or_legacy_index_is_reused_on_restart() -> anyhow::Result<()> {
+ let dir = tempfile::tempdir()?;
+ let index_dir = dir.path().join("index");
+ let kb = Uuid::now_v7();
+ let docs = [Uuid::now_v7(), Uuid::now_v7()];
+ let search = open(&index_dir)?;
+ rebuild(&index_dir, &search, rows(kb, docs))?;
+ assert!(!marker(&index_dir).exists());
+ drop(search);
+
+ let search = open(&index_dir)?;
+ assert_eq!(search.len(), 3);
+ // A legacy index has no marker either: ordinary writes and the next
+ // open must not erase it or turn a normal startup into a rebuild.
+ search.reindex_document(
+ &kb.to_string(),
+ &docs[0].to_string(),
+ &[(Uuid::now_v7().to_string(), "legacyneedle".into())],
+ )?;
+ drop(search);
+ let search = open(&index_dir)?;
+ assert_eq!(search.len(), 2);
+ assert_eq!(search.search(&kb.to_string(), "legacyneedle", 10)?.len(), 1);
+
+ // The database may become empty between attempts. Completing that
+ // recovery must clear its marker instead of rebuilding on every boot.
+ File::create(marker(&index_dir))?.sync_all()?;
+ drop(search);
+ let search = open(&index_dir)?;
+ assert!(search.is_empty());
+ rebuild(&index_dir, &search, Vec::new())?;
+ assert!(!marker(&index_dir).exists());
+ Ok(())
+ }
+}