Skip to content

World's first open-source enterprise world model.

9.6 KB
Raw Blame
//! utopia-search: 内嵌 Tantivy 全文索引(jieba 中文分词)+ RRF 融合。
//! 单索引多 KB:kb_id 作过滤字段;chunk 正文在 Postgres,索引里只存 id 映射。

mod docs;
pub use docs::{DocsIndex, DocsSection};

use std::collections::HashMap;
use std::path::Path;
use std::sync::Mutex;

use anyhow::Context;
use tantivy::collector::TopDocs;
use tantivy::query::{BooleanQuery, Occur, Query, TermQuery};
use tantivy::schema::{
    Field, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Value, STORED, STRING,
};
use tantivy::tokenizer::TextAnalyzer;
use tantivy::{Index, IndexReader, IndexWriter, TantivyDocument, Term};

const JIEBA: &str = "jieba";
const WRITER_HEAP: usize = 64 * 1024 * 1024;

pub struct SearchIndex {
    writer: Mutex<IndexWriter>,
    reader: IndexReader,
    analyzer: TextAnalyzer,
    f_chunk_id: Field,
    f_kb_id: Field,
    f_document_id: Field,
    f_text: Field,
}

#[derive(Debug, Clone)]
pub struct Hit {
    pub chunk_id: String,
    pub score: f32,
}

impl SearchIndex {
    pub fn open(dir: &Path) -> anyhow::Result<Self> {
        std::fs::create_dir_all(dir)?;

        let mut schema_builder = Schema::builder();
        let text_indexing = TextFieldIndexing::default()
            .set_tokenizer(JIEBA)
            .set_index_option(IndexRecordOption::WithFreqsAndPositions);
        let f_chunk_id = schema_builder.add_text_field("chunk_id", STRING | STORED);
        let f_kb_id = schema_builder.add_text_field("kb_id", STRING);
        let f_document_id = schema_builder.add_text_field("document_id", STRING);
        let f_text = schema_builder.add_text_field(
            "text",
            TextOptions::default().set_indexing_options(text_indexing),
        );
        let schema = schema_builder.build();

        let mmap =
            tantivy::directory::MmapDirectory::open(dir).context("Failed to open index dir")?;
        let index =
            Index::open_or_create(mmap, schema).context("Failed to open/create Tantivy index")?;
        index
            .tokenizers()
            .register(JIEBA, tantivy_jieba::JiebaTokenizer::new());

        let writer = index.writer(WRITER_HEAP)?;
        let reader = index.reader()?;
        let analyzer = index
            .tokenizers()
            .get(JIEBA)
            .context("jieba tokenizer 未注册")?;

        Ok(Self {
            writer: Mutex::new(writer),
            reader,
            analyzer,
            f_chunk_id,
            f_kb_id,
            f_document_id,
            f_text,
        })
    }

    /// 重建某文档的索引条目(先删后加,幂等),随后 commit。
    pub fn reindex_document(
        &self,
        kb_id: &str,
        document_id: &str,
        chunks: &[(String, String)],
    ) -> anyhow::Result<()> {
        let mut writer = self.writer.lock().expect("writer 锁中毒");
        writer.delete_term(Term::from_field_text(self.f_document_id, document_id));
        for (chunk_id, text) in chunks {
            let mut doc = TantivyDocument::default();
            doc.add_text(self.f_chunk_id, chunk_id);
            doc.add_text(self.f_kb_id, kb_id);
            doc.add_text(self.f_document_id, document_id);
            doc.add_text(self.f_text, text);
            writer.add_document(doc)?;
        }
        writer.commit()?;
        // 写后即可读:不等 reader 的异步重载窗口
        self.reader.reload()?;
        Ok(())
    }

    /// Discard an incomplete rebuild while retaining the index's writer lock.
    /// Removing its directory first could erase another process's live index.
    pub fn clear(&self) -> anyhow::Result<()> {
        let mut writer = self.writer.lock().expect("writer 锁中毒");
        writer.delete_all_documents()?;
        writer.commit()?;
        self.reader.reload()?;
        Ok(())
    }

    pub fn delete_document(&self, document_id: &str) -> anyhow::Result<()> {
        let mut writer = self.writer.lock().expect("writer 锁中毒");
        writer.delete_term(Term::from_field_text(self.f_document_id, document_id));
        writer.commit()?;
        self.reader.reload()?;
        Ok(())
    }

    /// 索引里有多少条分块。**启动时拿它跟库里的数对账**——索引目录是独立于
    /// 数据库的一份文件(换机器、卷没挂上、损坏都可能让它落空),而落空之后
    /// 检索只会静默回零,界面上看不出任何异样。
    pub fn len(&self) -> usize {
        self.reader.searcher().num_docs() as usize
    }

    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    /// BM25 检索(限定 kb)。
    /// 查询用与索引完全一致的 jieba analyzer 切词后按 OR 组合——
    /// 不能走 QueryParser:CJK 整句会被当成短语查询(要求词连续出现),召回归零。
    pub fn search(&self, kb_id: &str, query: &str, limit: usize) -> anyhow::Result<Vec<Hit>> {
        let mut analyzer = self.analyzer.clone();
        let mut stream = analyzer.token_stream(query);
        let mut term_queries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
        while stream.advance() {
            let token = stream.token();
            let word = token.text.trim();
            if word.is_empty() || word.chars().all(|c| !c.is_alphanumeric()) {
                continue;
            }
            term_queries.push((
                Occur::Should,
                Box::new(TermQuery::new(
                    Term::from_field_text(self.f_text, word),
                    IndexRecordOption::WithFreqs,
                )),
            ));
        }
        if term_queries.is_empty() {
            return Ok(Vec::new());
        }

        let kb_filter: Box<dyn Query> = Box::new(TermQuery::new(
            Term::from_field_text(self.f_kb_id, kb_id),
            IndexRecordOption::Basic,
        ));
        let text_query: Box<dyn Query> = Box::new(BooleanQuery::new(term_queries));
        let combined = BooleanQuery::new(vec![(Occur::Must, kb_filter), (Occur::Must, text_query)]);

        let searcher = self.reader.searcher();
        let collector = TopDocs::with_limit(limit).order_by_score();
        let top = searcher.search(&combined, &collector)?;
        let mut hits = Vec::with_capacity(top.len());
        for (score, addr) in top {
            let doc: TantivyDocument = searcher.doc(addr)?;
            if let Some(chunk_id) = doc.get_first(self.f_chunk_id).and_then(|v| v.as_str()) {
                hits.push(Hit {
                    chunk_id: chunk_id.to_string(),
                    score,
                });
            }
        }
        Ok(hits)
    }
}

/// Reciprocal Rank Fusion:融合多路召回的排名(k=60 为经验常数)。
pub fn rrf_fuse(lists: &[Vec<String>], limit: usize) -> Vec<String> {
    const K: f64 = 60.0;
    let mut scores: HashMap<String, f64> = HashMap::new();
    for list in lists {
        for (rank, id) in list.iter().enumerate() {
            *scores.entry(id.clone()).or_default() += 1.0 / (K + rank as f64 + 1.0);
        }
    }
    let mut ranked: Vec<(String, f64)> = scores.into_iter().collect();
    // 同分按 id 定序,否则 HashMap 的随机顺序会改变上限内保留的证据和口径。
    ranked.sort_by(|a, b| {
        b.1.partial_cmp(&a.1)
            .unwrap_or(std::cmp::Ordering::Equal)
            .then_with(|| a.0.cmp(&b.0))
    });
    ranked.into_iter().take(limit).map(|(id, _)| id).collect()
}

#[cfg(test)]
mod rrf_tests {
    use super::rrf_fuse;

    fn ids(list: &[&str]) -> Vec<String> {
        list.iter().map(|s| s.to_string()).collect()
    }

    /// 一路召回就是那一路的顺序,融合不许重排
    #[test]
    fn one_list_comes_back_in_its_own_order() {
        let lists = vec![ids(&["a", "b", "c"])];
        assert_eq!(rrf_fuse(&lists, 10), ids(&["a", "b", "c"]));
    }

    /// 两路都点名的排最前;只在一路里的按各自名次的倒数相加比大小:
    /// a = 1/61 + 1/62,c = 1/63 + 1/61,b = 1/62
    #[test]
    fn two_lists_follow_the_reciprocal_rank_arithmetic() {
        let lists = vec![ids(&["a", "b", "c"]), ids(&["c", "a"])];
        assert_eq!(rrf_fuse(&lists, 10), ids(&["a", "c", "b"]));
    }

    /// 空的那一路不是失败的那一路:它什么分都不加,也不影响别人
    #[test]
    fn an_empty_list_contributes_nothing() {
        let lists = vec![ids(&["a", "b"]), Vec::new()];
        assert_eq!(rrf_fuse(&lists, 10), ids(&["a", "b"]));
    }

    /// 上限在融合之后裁,不是每路各裁
    #[test]
    fn the_limit_cuts_after_fusion() {
        let lists = vec![ids(&["a", "b"]), ids(&["b", "c"])];
        assert_eq!(rrf_fuse(&lists, 1), ids(&["b"]));
    }

    #[test]
    fn equal_scores_keep_the_same_candidates_at_every_limit() {
        let mut lists = vec![ids(&["z", "b"]), ids(&["y", "a"])];
        let expected = ids(&["y", "z", "a", "b"]);
        // Each call builds a newly seeded HashMap. Both channel heads must
        // precede their runners-up, and a cutoff through either tie is stable.
        for _ in 0..32 {
            for limit in 0..=expected.len() + 1 {
                assert_eq!(
                    rrf_fuse(&lists, limit),
                    expected[..limit.min(expected.len())]
                );
            }
            lists.reverse();
        }
    }

    #[test]
    fn equal_scores_from_opposite_ranks_are_ordered_by_id() {
        let lists = vec![ids(&["c", "a", "b"]), ids(&["b", "a", "c"])];
        // b and c each score 1/61 + 1/63, above a's 2/62. Equal totals
        // must not acquire a preference for whichever channel came first.
        for _ in 0..32 {
            assert_eq!(rrf_fuse(&lists, 3), ids(&["b", "c", "a"]));
            assert_eq!(rrf_fuse(&lists, 1), ids(&["b"]));
        }
    }
}