Skip to content

World's first open-source enterprise world model.

18.2 KB
Raw Blame
//! 摄入管道:parse → chunk → 全文索引 → embedding(可选)→ ready。
//! 每步幂等:重跑会先清掉旧分块与旧索引条目。

use crate::llm_util;
use crate::state::AppState;
use futures_util::{stream, StreamExt};
use utopia_core::models::{LlmSettings, Proposer};
use utopia_llm::LlmClient;
use utopia_store::documents::{self, ProcessingInput};
use uuid::Uuid;

/// 这份文档的来源要不要抽取。没有来源的文档(直接上传、记忆片段)照旧抽。
/// 文档所属来源的种类;没有来源(上传)为 None。抽取那边也要问同一个问题(0054)
pub(crate) async fn source_kind(
    state: &AppState,
    source_id: Option<Uuid>,
) -> anyhow::Result<Option<String>> {
    let Some(id) = source_id else {
        return Ok(None);
    };
    Ok(Some(
        utopia_store::sources::get(&state.pool, id).await?.kind,
    ))
}

async fn source_extracts(state: &AppState, source_id: Option<Uuid>) -> anyhow::Result<bool> {
    let Some(id) = source_id else {
        return Ok(true);
    };
    Ok(utopia_store::sources::get(&state.pool, id)
        .await?
        .extracts())
}

/// 每次送多少条去嵌入。
///
/// 与 ontology_index 的 64 不同,这里没量过,先不动:那边的注释说批大小要拿真实文本量,
/// 块文本比类标签长得多,16 未必错。并发与批大小两个旋钮别在一刀里同时拧,否则结果读不出来。
const EMBED_BATCH: usize = 16;

/// 同时在飞的嵌入批数上限(#513)。
///
/// 嵌入模型的并发闸门(`model_concurrency`,缺省 10)是部署级共享的:本体补齐
/// (ontology_index,上限 4)、类型消解、每次提问的查询嵌入都走它。摄入是用户等着看的
/// 前台活,比后台补齐更有资格用闸门;但它也是能一次带来上千个分块的那一路,不设上限
/// 就把补齐、消解、检索全饿死——ontology_index 记着一次 `join_all` 把闸门占死、五篇文档
/// 卡在 embedding 十五分钟的事故。4 与补齐相同:两路同时跑合计 8,闸门还剩 2 给别人。
const EMBED_JOBS: usize = 4;

pub async fn process_document(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
    let input = documents::processing_input(&state.pool, document_id).await?;
    let doc = &input.document;
    match run(state, &input).await {
        Ok(()) => Ok(()),
        // 在等读字的服务读完:文档照旧是 parsing,任务过一会儿再来问
        Err(e) if utopia_core::is_deferred(&e).is_some() => {
            if documents::processing_is_current(&state.pool, &input).await? {
                Err(e)
            } else {
                Ok(())
            }
        }
        Err(e) => {
            // 字要靠模型读、而那种模型没配(0040):**降级**。文件留着,文档停在 failed
            // 并记下缺哪一种,报一条库级告警;配上模型会自己重新处理。重试没用——
            // 模型不会在两分钟里自己配上,所以挂 Terminal。
            // 转写回来却分不出说话人,跟没配一样对待(决定 5):换个会标说话人的模型再来
            let waiting = e
                .downcast_ref::<utopia_ingest::NeedsReader>()
                .map(|n| n.reader)
                .or_else(|| {
                    e.downcast_ref::<utopia_ingest::NoSpeakers>()
                        .map(|_| utopia_ingest::Reader::Transcribe)
                });
            if !documents::set_failed_if_current(
                &state.pool,
                &input,
                &e.to_string(),
                waiting.map(|reader| reader.as_str()),
            )
            .await?
            {
                // 过期模型结果可丢弃,数据库故障仍要让队列看见。
                if e.is::<sqlx::Error>()
                    || matches!(
                        e.downcast_ref::<utopia_core::AppError>(),
                        Some(utopia_core::AppError::Db(_))
                    )
                {
                    return Err(e);
                }
                tracing::info!(%document_id, "discarding a superseded document failure");
                return Ok(());
            }
            if let Some(reader) = waiting {
                crate::alerting::observe_document_needs_reader(
                    state,
                    doc.kb_id,
                    document_id,
                    &doc.filename,
                    reader,
                    &e.to_string(),
                )
                .await;
                state.emit_document(doc.kb_id, document_id);
                return Err(e.context(utopia_core::Terminal));
            }
            state.emit_document(doc.kb_id, document_id);
            // 读不了的格式(视频、二进制、空文件)换多少次也一样
            if e.downcast_ref::<utopia_ingest::Unreadable>().is_some() {
                return Err(e.context(utopia_core::Terminal));
            }
            Err(e)
        }
    }
}

async fn run(state: &AppState, input: &ProcessingInput) -> anyhow::Result<()> {
    let doc = &input.document;
    let document_id = doc.id;
    // 排队之后被删了(#268):墓碑不重建分块、不回索引;清过的连原文都没了
    if doc.deleted_at.is_some() {
        tracing::info!(document = %document_id, "skipping a deleted document");
        return Ok(());
    }

    // 1. 解析(CPU 密集,放 blocking 线程)
    if !documents::set_status_if_current(&state.pool, input, "parsing").await? {
        return Ok(());
    }
    state.emit_document(doc.kb_id, document_id);
    let bytes = state.blob.get(&doc.sha256).await?;
    let filename = doc.filename.clone();
    let (parsed, bytes) =
        tokio::task::spawn_blocking(move || (utopia_ingest::parse(&filename, &bytes), bytes))
            .await?;
    let kb_row = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;
    let settings = utopia_store::settings::get(&state.pool, kb_row.workspace_id).await?;
    let pushed_statements =
        source_kind(state, doc.source_id).await?.as_deref() == Some("statements");

    // 2. 分块 + 入库
    let (text, pieces, warnings) = match parsed {
        Ok(parsed) => {
            let warnings = parsed.warnings;
            // 解析出来的正文可能夹着 NUL(PDF 文本层常见),入库之前剥掉(#611)——与记忆
            // 那条路共用 `utopia_core::without_nul`(#665)。剥必须在算长度、分块之前:之后的
            // text_len、分块偏移、全文索引、嵌入读的都是这一份,彼此才对得上
            let text = utopia_core::without_nul(&parsed.text).into_owned();
            // 推送来的陈述(0054):载荷就是契约,整份是一块。分块预算是给模型的注意力
            // 定的,这条路没有模型读;切开了契约就解析不回来
            let pieces = if pushed_statements {
                vec![utopia_ingest::ChunkPiece {
                    seq: 0,
                    char_start: 0,
                    char_end: text.chars().count() as i32,
                    heading: None,
                    provenance: utopia_ingest::Provenance::stated(),
                    text: text.clone(),
                }]
            } else {
                utopia_ingest::chunk_with_budget(&text, state.chunk_tokens)
            };
            (text, pieces, warnings)
        }
        // 没有文本层的扫描件、图片:工作区配了版面识别服务就交给它读(0040 第二刀),
        // 按页分段切块,每块记着页码和框。录音交给会标说话人的转写模型(第三刀),
        // 每块记着起止时刻和说话人。没配的照旧往上抛,由外面降级
        Err(e) => {
            let Some(needs) = e.downcast_ref::<utopia_ingest::NeedsReader>() else {
                return Err(e);
            };
            let reading = match (needs.reader, settings.as_ref()) {
                (utopia_ingest::Reader::Ocr, Some(s)) => {
                    match crate::readers::Ocr::from_settings(s) {
                        Some(ocr) => ocr.read(state, doc, bytes).await?,
                        None => return Err(e),
                    }
                }
                (utopia_ingest::Reader::Transcribe, Some(s)) => {
                    match crate::readers::Transcriber::from_settings(s) {
                        Some(t) => t.read(doc, bytes).await?,
                        None => return Err(e),
                    }
                }
                (_, None) => return Err(e),
            };
            if reading.text.trim().is_empty() {
                return Err(utopia_ingest::Unreadable(
                    "The reader found no text in this file".into(),
                )
                .into());
            }
            let pieces = reading.chunk(state.chunk_tokens);
            (reading.text, pieces, Vec::new())
        }
    };
    let text_len = text.chars().count() as i32;
    let Some(chunk_pairs) =
        documents::replace_chunks_if_current(&state.pool, input, &pieces).await?
    else {
        // 读取期间源文档可能已更新或删除,丢弃过期结果,
        // 不再改写新任务的索引、状态和抽取队列。
        tracing::info!(%document_id, "discarding a superseded document read");
        return Ok(());
    };
    let chunk_count = chunk_pairs.len() as i32;
    // Only the current version gets an alert: if this read was superseded, the warning is stale.
    if !warnings.is_empty() {
        crate::alerting::observe_document_truncated(
            state,
            doc.kb_id,
            document_id,
            &doc.filename,
            &warnings,
        )
        .await;
    }

    // 3. 全文索引(Tantivy)
    if !documents::set_status_if_current(&state.pool, input, "indexing").await? {
        return Ok(());
    }
    state.emit_document(doc.kb_id, document_id);
    let search = state.search.clone();
    let kb = doc.kb_id.to_string();
    let did = document_id.to_string();
    tokio::task::spawn_blocking(move || search.reindex_document(&kb, &did, &chunk_pairs)).await??;

    // 4. embedding(工作区配置了 embedding 模型才做;没配也算 ready,先享受 BM25 搜索)
    if let Some((settings, client)) = embedder(settings.as_ref()) {
        if !documents::set_status_if_current(&state.pool, input, "embedding").await? {
            return Ok(());
        }
        state.emit_document(doc.kb_id, document_id);
        embed_pending(state, settings, &client, input).await?;
    }

    // 来源说了不抽取的,到这里为止:可搜、可问,不进图。
    //
    // schema 文档就是这一类(0035 决定 7)——它是给问数检索表结构的语料,进抽取的
    // 结果是抽取器把每个列名当成一个实体(宽表语料上四十个概念实体里二十八个是
    // 列名,#553)。状态记成 `skipped` 而不是留在 `none`:`none` 在 Library 里读作
    // 「还没排到」,而它永远不会排到
    // 两段式:索引就绪后,若配置了对话模型则排队图谱抽取(不阻塞可搜可问)。
    // 推送来的陈述不问模型(0054),没配也排
    let graph_status = if !source_extracts(state, doc.source_id).await? {
        Some("skipped")
    } else if pushed_statements || settings.as_ref().is_some_and(|s| s.chat_ready()) {
        Some("queued")
    } else {
        None
    };
    if !documents::finish_processing_if_current(
        &state.pool,
        input,
        text_len,
        chunk_count,
        graph_status,
    )
    .await?
    {
        tracing::info!(%document_id, "discarding a superseded document completion");
        return Ok(());
    }
    state.emit_document(doc.kb_id, document_id);

    tracing::info!(%document_id, chunks = chunk_count, "文档处理完成");
    Ok(())
}

/// 记忆摄入(episodes 快速路径的后半程):新 episode chunk 补 embedding、
/// 重建全文索引、触发增量抽取(extracted_at 为空的新 chunk 才会被抽)。
/// 免解析免分块——episode 落库时已是 chunk。
///
/// `proposer`:说这句话的人,以及经 MCP 时那个 agent。一路传到抽取,落在
/// `pending_facts.proposed_by` / `proposed_token`(0015、0026)
pub async fn memory_ingest(
    state: &AppState,
    document_id: Uuid,
    proposer: Proposer,
) -> anyhow::Result<()> {
    let doc = utopia_store::documents::get(&state.pool, document_id).await?;
    if doc.deleted_at.is_some() {
        tracing::info!(document = %document_id, "skipping a deleted document");
        return Ok(());
    }
    let kb_row = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;
    let settings = utopia_store::settings::get(&state.pool, kb_row.workspace_id).await?;

    // 一句一句嵌,嵌不了的那句留着没有向量,别的照常往下走。日志是一篇越写越长的文档,
    // 从前所有没向量的块一起嵌、一处出错整个任务失败:一句嵌不了的记忆从此挡在每一句
    // 新记忆前面,索引不重建、抽取不排队,而 `remember` 照样回「记下了」(#1187)
    if let Some((settings, client)) = embedder(settings.as_ref()) {
        embed_each(state, settings, &client, document_id).await?;
    }

    let chunks = utopia_store::documents::chunks_full(&state.pool, document_id).await?;
    let pairs: Vec<(String, String)> = chunks
        .iter()
        .map(|c| (c.id.to_string(), c.text.clone()))
        .collect();
    let search = state.search.clone();
    let kb = doc.kb_id.to_string();
    let did = document_id.to_string();
    tokio::task::spawn_blocking(move || search.reindex_document(&kb, &did, &pairs)).await??;

    if settings.as_ref().is_some_and(|s| s.chat_ready()) {
        utopia_store::documents::set_graph_status(&state.pool, document_id, "queued").await?;
        utopia_store::jobs::enqueue(
            &state.pool,
            "extract_document",
            serde_json::json!({
                "document_id": document_id,
                "proposed_by": proposer.user_id,
                "proposed_token": proposer.token_id,
            }),
        )
        .await?;
    }
    state.emit_document(doc.kb_id, document_id);
    Ok(())
}

/// 记忆日志的嵌入:每块单独一次请求,失败的只记日志、留着下次再试,返回嵌成了几条。
/// 与 [`embed_pending`] 相反的取舍——那边一批错位会把向量写到别人身上,所以整篇失败;
/// 这里一次一条,没有错位可言,而一条失败不该连累后来的
async fn embed_each(
    state: &AppState,
    settings: &LlmSettings,
    client: &LlmClient,
    document_id: Uuid,
) -> anyhow::Result<usize> {
    let pending =
        utopia_store::documents::chunks_pending_embedding(&state.pool, document_id).await?;
    let mut done = 0;
    for (chunk_id, text) in pending {
        let embedded = {
            let _permit = llm_util::acquire_embed(state, settings).await;
            client.embed(std::slice::from_ref(&text)).await
        };
        match embedded {
            Ok(mut vectors) if vectors.len() == 1 => {
                let vector = vectors.remove(0);
                utopia_store::documents::set_embeddings(&state.pool, &[(chunk_id, vector)]).await?;
                done += 1;
            }
            Ok(vectors) => {
                tracing::warn!(%document_id, %chunk_id, got = vectors.len(), "记忆的向量数量对不上,这一句先不嵌");
            }
            Err(e) => {
                tracing::warn!(%document_id, %chunk_id, error = %e, "这一句记忆没嵌成,别的照常");
            }
        }
    }
    Ok(done)
}

/// 工作区配了嵌入模型才有客户端;设置与客户端一起交出去,闸门许可证要按设置取。
fn embedder(settings: Option<&LlmSettings>) -> Option<(&LlmSettings, LlmClient)> {
    let s = settings?;
    llm_util::embed_client(s).map(|c| (s, c))
}

/// 把这篇文档还没有向量的分块全部嵌完,返回嵌了多少条。文档摄入与记忆摄入共用——
/// 从前两处各抄一份同样的循环,且都是一批等完再发下一批。
///
/// 批与批之间互不依赖(向量按位置配对,只在一批之内),`EMBED_JOBS` 个批次同时在飞,
/// 每个批次握一张闸门许可证只到自己写完。
///
/// **数量对不上就整批放弃。** 配对是按位置的,少一条就全体错位,把一条的向量写到另一条
/// 身上;这种错落库之后再也看不出来——正文还在,向量是别人的。
///
/// **任一批失败整篇失败。** 串行时错误自然一路 unwind;并发下要明确地把它传出去,让调用方
/// 把文档标成 failed,而不是留它停在 embedding。已经写进去的向量留着,重跑只补缺的
/// (`chunks_pending_embedding` 只挑 embedding 为空的)。
async fn embed_pending(
    state: &AppState,
    settings: &LlmSettings,
    client: &LlmClient,
    input: &ProcessingInput,
) -> anyhow::Result<usize> {
    let pending = documents::chunks_pending_embedding(&state.pool, input.document.id).await?;
    // 批次先切成拥有的数据:借来的切片捕进并发驱动的 future 过不了 Send 边界
    //(ontology_index 里同一句话)
    let batches: Vec<Vec<(Uuid, String)>> =
        pending.chunks(EMBED_BATCH).map(<[_]>::to_vec).collect();
    let mut batches = stream::iter(batches)
        .map(|batch| async move {
            let texts: Vec<String> = batch.iter().map(|(_, t)| t.clone()).collect();
            let _permit = llm_util::acquire_embed(state, settings).await;
            let embeddings = client.embed(&texts).await?;
            if embeddings.len() != batch.len() {
                anyhow::bail!(
                    "嵌入返回 {} 条,送去的是 {} 条",
                    embeddings.len(),
                    batch.len()
                );
            }
            let items: Vec<(Uuid, Vec<f32>)> =
                batch.iter().map(|(id, _)| *id).zip(embeddings).collect();
            if !documents::set_embeddings_if_current(&state.pool, input, &items).await? {
                return Ok(0);
            }
            Ok::<usize, anyhow::Error>(items.len())
        })
        .buffer_unordered(EMBED_JOBS);
    let mut done = 0;
    while let Some(written) = batches.next().await {
        done += written?;
    }
    Ok(done)
}

#[cfg(test)]
#[path = "pipeline_tests.rs"]
mod pipeline_tests;