World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
//! 摄入管道:parse → chunk → 全文索引 → embedding(可选)→ ready。
//! 每步幂等:重跑会先清掉旧分块与旧索引条目。
use crate::llm_util;
use crate::state::AppState;
use futures_util::{stream, StreamExt};
use utopia_core::models::{LlmSettings, Proposer};
use utopia_llm::LlmClient;
use utopia_store::documents::{self, ProcessingInput};
use uuid::Uuid;
/// 这份文档的来源要不要抽取。没有来源的文档(直接上传、记忆片段)照旧抽。
/// 文档所属来源的种类;没有来源(上传)为 None。抽取那边也要问同一个问题(0054)
pub(crate) async fn source_kind(
state: &AppState,
source_id: Option<Uuid>,
) -> anyhow::Result<Option<String>> {
let Some(id) = source_id else {
return Ok(None);
};
Ok(Some(
utopia_store::sources::get(&state.pool, id).await?.kind,
))
}
async fn source_extracts(state: &AppState, source_id: Option<Uuid>) -> anyhow::Result<bool> {
let Some(id) = source_id else {
return Ok(true);
};
Ok(utopia_store::sources::get(&state.pool, id)
.await?
.extracts())
}
/// 每次送多少条去嵌入。
///
/// 与 ontology_index 的 64 不同,这里没量过,先不动:那边的注释说批大小要拿真实文本量,
/// 块文本比类标签长得多,16 未必错。并发与批大小两个旋钮别在一刀里同时拧,否则结果读不出来。
const EMBED_BATCH: usize = 16;
/// 同时在飞的嵌入批数上限(#513)。
///
/// 嵌入模型的并发闸门(`model_concurrency`,缺省 10)是部署级共享的:本体补齐
/// (ontology_index,上限 4)、类型消解、每次提问的查询嵌入都走它。摄入是用户等着看的
/// 前台活,比后台补齐更有资格用闸门;但它也是能一次带来上千个分块的那一路,不设上限
/// 就把补齐、消解、检索全饿死——ontology_index 记着一次 `join_all` 把闸门占死、五篇文档
/// 卡在 embedding 十五分钟的事故。4 与补齐相同:两路同时跑合计 8,闸门还剩 2 给别人。
const EMBED_JOBS: usize = 4;
pub async fn process_document(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
let input = documents::processing_input(&state.pool, document_id).await?;
let doc = &input.document;
match run(state, &input).await {
Ok(()) => Ok(()),
// 在等读字的服务读完:文档照旧是 parsing,任务过一会儿再来问
Err(e) if utopia_core::is_deferred(&e).is_some() => {
if documents::processing_is_current(&state.pool, &input).await? {
Err(e)
} else {
Ok(())
}
}
Err(e) => {
// 字要靠模型读、而那种模型没配(0040):**降级**。文件留着,文档停在 failed
// 并记下缺哪一种,报一条库级告警;配上模型会自己重新处理。重试没用——
// 模型不会在两分钟里自己配上,所以挂 Terminal。
// 转写回来却分不出说话人,跟没配一样对待(决定 5):换个会标说话人的模型再来
let waiting = e
.downcast_ref::<utopia_ingest::NeedsReader>()
.map(|n| n.reader)
.or_else(|| {
e.downcast_ref::<utopia_ingest::NoSpeakers>()
.map(|_| utopia_ingest::Reader::Transcribe)
});
if !documents::set_failed_if_current(
&state.pool,
&input,
&e.to_string(),
waiting.map(|reader| reader.as_str()),
)
.await?
{
// 过期模型结果可丢弃,数据库故障仍要让队列看见。
if e.is::<sqlx::Error>()
|| matches!(
e.downcast_ref::<utopia_core::AppError>(),
Some(utopia_core::AppError::Db(_))
)
{
return Err(e);
}
tracing::info!(%document_id, "discarding a superseded document failure");
return Ok(());
}
if let Some(reader) = waiting {
crate::alerting::observe_document_needs_reader(
state,
doc.kb_id,
document_id,
&doc.filename,
reader,
&e.to_string(),
)
.await;
state.emit_document(doc.kb_id, document_id);
return Err(e.context(utopia_core::Terminal));
}
state.emit_document(doc.kb_id, document_id);
// 读不了的格式(视频、二进制、空文件)换多少次也一样
if e.downcast_ref::<utopia_ingest::Unreadable>().is_some() {
return Err(e.context(utopia_core::Terminal));
}
Err(e)
}
}
}
async fn run(state: &AppState, input: &ProcessingInput) -> anyhow::Result<()> {
let doc = &input.document;
let document_id = doc.id;
// 排队之后被删了(#268):墓碑不重建分块、不回索引;清过的连原文都没了
if doc.deleted_at.is_some() {
tracing::info!(document = %document_id, "skipping a deleted document");
return Ok(());
}
// 1. 解析(CPU 密集,放 blocking 线程)
if !documents::set_status_if_current(&state.pool, input, "parsing").await? {
return Ok(());
}
state.emit_document(doc.kb_id, document_id);
let bytes = state.blob.get(&doc.sha256).await?;
let filename = doc.filename.clone();
let (parsed, bytes) =
tokio::task::spawn_blocking(move || (utopia_ingest::parse(&filename, &bytes), bytes))
.await?;
let kb_row = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;
let settings = utopia_store::settings::get(&state.pool, kb_row.workspace_id).await?;
let pushed_statements =
source_kind(state, doc.source_id).await?.as_deref() == Some("statements");
// 2. 分块 + 入库
let (text, pieces, warnings) = match parsed {
Ok(parsed) => {
let warnings = parsed.warnings;
// 解析出来的正文可能夹着 NUL(PDF 文本层常见),入库之前剥掉(#611)——与记忆
// 那条路共用 `utopia_core::without_nul`(#665)。剥必须在算长度、分块之前:之后的
// text_len、分块偏移、全文索引、嵌入读的都是这一份,彼此才对得上
let text = utopia_core::without_nul(&parsed.text).into_owned();
// 推送来的陈述(0054):载荷就是契约,整份是一块。分块预算是给模型的注意力
// 定的,这条路没有模型读;切开了契约就解析不回来
let pieces = if pushed_statements {
vec![utopia_ingest::ChunkPiece {
seq: 0,
char_start: 0,
char_end: text.chars().count() as i32,
heading: None,
provenance: utopia_ingest::Provenance::stated(),
text: text.clone(),
}]
} else {
utopia_ingest::chunk_with_budget(&text, state.chunk_tokens)
};
(text, pieces, warnings)
}
// 没有文本层的扫描件、图片:工作区配了版面识别服务就交给它读(0040 第二刀),
// 按页分段切块,每块记着页码和框。录音交给会标说话人的转写模型(第三刀),
// 每块记着起止时刻和说话人。没配的照旧往上抛,由外面降级
Err(e) => {
let Some(needs) = e.downcast_ref::<utopia_ingest::NeedsReader>() else {
return Err(e);
};
let reading = match (needs.reader, settings.as_ref()) {
(utopia_ingest::Reader::Ocr, Some(s)) => {
match crate::readers::Ocr::from_settings(s) {
Some(ocr) => ocr.read(state, doc, bytes).await?,
None => return Err(e),
}
}
(utopia_ingest::Reader::Transcribe, Some(s)) => {
match crate::readers::Transcriber::from_settings(s) {
Some(t) => t.read(doc, bytes).await?,
None => return Err(e),
}
}
(_, None) => return Err(e),
};
if reading.text.trim().is_empty() {
return Err(utopia_ingest::Unreadable(
"The reader found no text in this file".into(),
)
.into());
}
let pieces = reading.chunk(state.chunk_tokens);
(reading.text, pieces, Vec::new())
}
};
let text_len = text.chars().count() as i32;
let Some(chunk_pairs) =
documents::replace_chunks_if_current(&state.pool, input, &pieces).await?
else {
// 读取期间源文档可能已更新或删除,丢弃过期结果,
// 不再改写新任务的索引、状态和抽取队列。
tracing::info!(%document_id, "discarding a superseded document read");
return Ok(());
};
let chunk_count = chunk_pairs.len() as i32;
// Only the current version gets an alert: if this read was superseded, the warning is stale.
if !warnings.is_empty() {
crate::alerting::observe_document_truncated(
state,
doc.kb_id,
document_id,
&doc.filename,
&warnings,
)
.await;
}
// 3. 全文索引(Tantivy)
if !documents::set_status_if_current(&state.pool, input, "indexing").await? {
return Ok(());
}
state.emit_document(doc.kb_id, document_id);
let search = state.search.clone();
let kb = doc.kb_id.to_string();
let did = document_id.to_string();
tokio::task::spawn_blocking(move || search.reindex_document(&kb, &did, &chunk_pairs)).await??;
// 4. embedding(工作区配置了 embedding 模型才做;没配也算 ready,先享受 BM25 搜索)
if let Some((settings, client)) = embedder(settings.as_ref()) {
if !documents::set_status_if_current(&state.pool, input, "embedding").await? {
return Ok(());
}
state.emit_document(doc.kb_id, document_id);
embed_pending(state, settings, &client, input).await?;
}
// 来源说了不抽取的,到这里为止:可搜、可问,不进图。
//
// schema 文档就是这一类(0035 决定 7)——它是给问数检索表结构的语料,进抽取的
// 结果是抽取器把每个列名当成一个实体(宽表语料上四十个概念实体里二十八个是
// 列名,#553)。状态记成 `skipped` 而不是留在 `none`:`none` 在 Library 里读作
// 「还没排到」,而它永远不会排到
// 两段式:索引就绪后,若配置了对话模型则排队图谱抽取(不阻塞可搜可问)。
// 推送来的陈述不问模型(0054),没配也排
let graph_status = if !source_extracts(state, doc.source_id).await? {
Some("skipped")
} else if pushed_statements || settings.as_ref().is_some_and(|s| s.chat_ready()) {
Some("queued")
} else {
None
};
if !documents::finish_processing_if_current(
&state.pool,
input,
text_len,
chunk_count,
graph_status,
)
.await?
{
tracing::info!(%document_id, "discarding a superseded document completion");
return Ok(());
}
state.emit_document(doc.kb_id, document_id);
tracing::info!(%document_id, chunks = chunk_count, "文档处理完成");
Ok(())
}
/// 记忆摄入(episodes 快速路径的后半程):新 episode chunk 补 embedding、
/// 重建全文索引、触发增量抽取(extracted_at 为空的新 chunk 才会被抽)。
/// 免解析免分块——episode 落库时已是 chunk。
///
/// `proposer`:说这句话的人,以及经 MCP 时那个 agent。一路传到抽取,落在
/// `pending_facts.proposed_by` / `proposed_token`(0015、0026)
pub async fn memory_ingest(
state: &AppState,
document_id: Uuid,
proposer: Proposer,
) -> anyhow::Result<()> {
let doc = utopia_store::documents::get(&state.pool, document_id).await?;
if doc.deleted_at.is_some() {
tracing::info!(document = %document_id, "skipping a deleted document");
return Ok(());
}
let kb_row = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;
let settings = utopia_store::settings::get(&state.pool, kb_row.workspace_id).await?;
// 一句一句嵌,嵌不了的那句留着没有向量,别的照常往下走。日志是一篇越写越长的文档,
// 从前所有没向量的块一起嵌、一处出错整个任务失败:一句嵌不了的记忆从此挡在每一句
// 新记忆前面,索引不重建、抽取不排队,而 `remember` 照样回「记下了」(#1187)
if let Some((settings, client)) = embedder(settings.as_ref()) {
embed_each(state, settings, &client, document_id).await?;
}
let chunks = utopia_store::documents::chunks_full(&state.pool, document_id).await?;
let pairs: Vec<(String, String)> = chunks
.iter()
.map(|c| (c.id.to_string(), c.text.clone()))
.collect();
let search = state.search.clone();
let kb = doc.kb_id.to_string();
let did = document_id.to_string();
tokio::task::spawn_blocking(move || search.reindex_document(&kb, &did, &pairs)).await??;
if settings.as_ref().is_some_and(|s| s.chat_ready()) {
utopia_store::documents::set_graph_status(&state.pool, document_id, "queued").await?;
utopia_store::jobs::enqueue(
&state.pool,
"extract_document",
serde_json::json!({
"document_id": document_id,
"proposed_by": proposer.user_id,
"proposed_token": proposer.token_id,
}),
)
.await?;
}
state.emit_document(doc.kb_id, document_id);
Ok(())
}
/// 记忆日志的嵌入:每块单独一次请求,失败的只记日志、留着下次再试,返回嵌成了几条。
/// 与 [`embed_pending`] 相反的取舍——那边一批错位会把向量写到别人身上,所以整篇失败;
/// 这里一次一条,没有错位可言,而一条失败不该连累后来的
async fn embed_each(
state: &AppState,
settings: &LlmSettings,
client: &LlmClient,
document_id: Uuid,
) -> anyhow::Result<usize> {
let pending =
utopia_store::documents::chunks_pending_embedding(&state.pool, document_id).await?;
let mut done = 0;
for (chunk_id, text) in pending {
let embedded = {
let _permit = llm_util::acquire_embed(state, settings).await;
client.embed(std::slice::from_ref(&text)).await
};
match embedded {
Ok(mut vectors) if vectors.len() == 1 => {
let vector = vectors.remove(0);
utopia_store::documents::set_embeddings(&state.pool, &[(chunk_id, vector)]).await?;
done += 1;
}
Ok(vectors) => {
tracing::warn!(%document_id, %chunk_id, got = vectors.len(), "记忆的向量数量对不上,这一句先不嵌");
}
Err(e) => {
tracing::warn!(%document_id, %chunk_id, error = %e, "这一句记忆没嵌成,别的照常");
}
}
}
Ok(done)
}
/// 工作区配了嵌入模型才有客户端;设置与客户端一起交出去,闸门许可证要按设置取。
fn embedder(settings: Option<&LlmSettings>) -> Option<(&LlmSettings, LlmClient)> {
let s = settings?;
llm_util::embed_client(s).map(|c| (s, c))
}
/// 把这篇文档还没有向量的分块全部嵌完,返回嵌了多少条。文档摄入与记忆摄入共用——
/// 从前两处各抄一份同样的循环,且都是一批等完再发下一批。
///
/// 批与批之间互不依赖(向量按位置配对,只在一批之内),`EMBED_JOBS` 个批次同时在飞,
/// 每个批次握一张闸门许可证只到自己写完。
///
/// **数量对不上就整批放弃。** 配对是按位置的,少一条就全体错位,把一条的向量写到另一条
/// 身上;这种错落库之后再也看不出来——正文还在,向量是别人的。
///
/// **任一批失败整篇失败。** 串行时错误自然一路 unwind;并发下要明确地把它传出去,让调用方
/// 把文档标成 failed,而不是留它停在 embedding。已经写进去的向量留着,重跑只补缺的
/// (`chunks_pending_embedding` 只挑 embedding 为空的)。
async fn embed_pending(
state: &AppState,
settings: &LlmSettings,
client: &LlmClient,
input: &ProcessingInput,
) -> anyhow::Result<usize> {
let pending = documents::chunks_pending_embedding(&state.pool, input.document.id).await?;
// 批次先切成拥有的数据:借来的切片捕进并发驱动的 future 过不了 Send 边界
//(ontology_index 里同一句话)
let batches: Vec<Vec<(Uuid, String)>> =
pending.chunks(EMBED_BATCH).map(<[_]>::to_vec).collect();
let mut batches = stream::iter(batches)
.map(|batch| async move {
let texts: Vec<String> = batch.iter().map(|(_, t)| t.clone()).collect();
let _permit = llm_util::acquire_embed(state, settings).await;
let embeddings = client.embed(&texts).await?;
if embeddings.len() != batch.len() {
anyhow::bail!(
"嵌入返回 {} 条,送去的是 {} 条",
embeddings.len(),
batch.len()
);
}
let items: Vec<(Uuid, Vec<f32>)> =
batch.iter().map(|(id, _)| *id).zip(embeddings).collect();
if !documents::set_embeddings_if_current(&state.pool, input, &items).await? {
return Ok(0);
}
Ok::<usize, anyhow::Error>(items.len())
})
.buffer_unordered(EMBED_JOBS);
let mut done = 0;
while let Some(written) = batches.next().await {
done += written?;
}
Ok(done)
}
#[cfg(test)]
#[path = "pipeline_tests.rs"]
mod pipeline_tests;