World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Ingest embeds four batches at a time, and both ingests share the loop (#539)
WaylandYang43d68d792246
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rsindex 096414a7..0ced3d11 100644--- a/crates/utopia-server/src/pipeline.rs+++ b/crates/utopia-server/src/pipeline.rs@@ -3,11 +3,26 @@use crate::llm_util;use crate::state::AppState;-use utopia_core::models::Proposer;+use futures_util::{stream, StreamExt};+use utopia_core::models::{LlmSettings, Proposer};+use utopia_llm::LlmClient;use uuid::Uuid;+/// 每次送多少条去嵌入。+///+/// 与 ontology_index 的 64 不同,这里没量过,先不动:那边的注释说批大小要拿真实文本量,+/// 块文本比类标签长得多,16 未必错。并发与批大小两个旋钮别在一刀里同时拧,否则结果读不出来。const EMBED_BATCH: usize = 16;+/// 同时在飞的嵌入批数上限(#513)。+///+/// 嵌入模型的并发闸门(`model_concurrency`,缺省 10)是部署级共享的:本体补齐+/// (ontology_index,上限 4)、类型消解、每次提问的查询嵌入都走它。摄入是用户等着看的+/// 前台活,比后台补齐更有资格用闸门;但它也是能一次带来上千个分块的那一路,不设上限+/// 就把补齐、消解、检索全饿死——ontology_index 记着一次 `join_all` 把闸门占死、五篇文档+/// 卡在 embedding 十五分钟的事故。4 与补齐相同:两路同时跑合计 8,闸门还剩 2 给别人。+const EMBED_JOBS: usize = 4;+pub async fn process_document(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {match run(state, document_id).await {Ok(()) => Ok(()),@@ -57,25 +72,10 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {// 4. embedding(工作区配置了 embedding 模型才做;没配也算 ready,先享受 BM25 搜索)let kb_row = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;let settings = utopia_store::settings::get(&state.pool, kb_row.workspace_id).await?;- if let Some(client) = settings.as_ref().and_then(llm_util::embed_client) {+ if let Some((settings, client)) = embedder(settings.as_ref()) {utopia_store::documents::set_status(&state.pool, document_id, "embedding").await?;state.emit_document(doc.kb_id, document_id);- let pending =- utopia_store::documents::chunks_pending_embedding(&state.pool, document_id).await?;- for batch in pending.chunks(EMBED_BATCH) {- let texts: Vec<String> = batch.iter().map(|(_, t)| t.clone()).collect();- let _permit = match settings.as_ref() {- Some(s) => llm_util::acquire_embed(state, s).await,- None => None,- };- let embeddings = client.embed(&texts).await?;- if embeddings.len() != batch.len() {- anyhow::bail!("Embedding 返回数量不匹配");- }- let items: Vec<(Uuid, Vec<f32>)> =- batch.iter().map(|(id, _)| *id).zip(embeddings).collect();- utopia_store::documents::set_embeddings(&state.pool, &items).await?;- }+ embed_pending(state, settings, &client, document_id).await?;}utopia_store::documents::set_ready(&state.pool, document_id, text_len, chunk_count).await?;@@ -115,23 +115,8 @@ pub async fn memory_ingest(let kb_row = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;let settings = utopia_store::settings::get(&state.pool, kb_row.workspace_id).await?;- if let Some(client) = settings.as_ref().and_then(llm_util::embed_client) {- let pending =- utopia_store::documents::chunks_pending_embedding(&state.pool, document_id).await?;- for batch in pending.chunks(EMBED_BATCH) {- let texts: Vec<String> = batch.iter().map(|(_, t)| t.clone()).collect();- let _permit = match settings.as_ref() {- Some(s) => llm_util::acquire_embed(state, s).await,- None => None,- };- let embeddings = client.embed(&texts).await?;- if embeddings.len() != batch.len() {- anyhow::bail!("Embedding 返回数量不匹配");- }- let items: Vec<(Uuid, Vec<f32>)> =- batch.iter().map(|(id, _)| *id).zip(embeddings).collect();- utopia_store::documents::set_embeddings(&state.pool, &items).await?;- }+ if let Some((settings, client)) = embedder(settings.as_ref()) {+ embed_pending(state, settings, &client, document_id).await?;}let chunks = utopia_store::documents::chunks_full(&state.pool, document_id).await?;@@ -160,3 +145,62 @@ pub async fn memory_ingest(state.emit_document(doc.kb_id, document_id);Ok(())}++/// 工作区配了嵌入模型才有客户端;设置与客户端一起交出去,闸门许可证要按设置取。+fn embedder(settings: Option<&LlmSettings>) -> Option<(&LlmSettings, LlmClient)> {+ let s = settings?;+ llm_util::embed_client(s).map(|c| (s, c))+}++/// 把这篇文档还没有向量的分块全部嵌完,返回嵌了多少条。文档摄入与记忆摄入共用——+/// 从前两处各抄一份同样的循环,且都是一批等完再发下一批。+///+/// 批与批之间互不依赖(向量按位置配对,只在一批之内),`EMBED_JOBS` 个批次同时在飞,+/// 每个批次握一张闸门许可证只到自己写完。+///+/// **数量对不上就整批放弃。** 配对是按位置的,少一条就全体错位,把一条的向量写到另一条+/// 身上;这种错落库之后再也看不出来——正文还在,向量是别人的。+///+/// **任一批失败整篇失败。** 串行时错误自然一路 unwind;并发下要明确地把它传出去,让调用方+/// 把文档标成 failed,而不是留它停在 embedding。已经写进去的向量留着,重跑只补缺的+/// (`chunks_pending_embedding` 只挑 embedding 为空的)。+async fn embed_pending(+ state: &AppState,+ settings: &LlmSettings,+ client: &LlmClient,+ document_id: Uuid,+) -> anyhow::Result<usize> {+ let pending =+ utopia_store::documents::chunks_pending_embedding(&state.pool, document_id).await?;+ // 批次先切成拥有的数据:借来的切片捕进并发驱动的 future 过不了 Send 边界+ //(ontology_index 里同一句话)+ let batches: Vec<Vec<(Uuid, String)>> =+ pending.chunks(EMBED_BATCH).map(<[_]>::to_vec).collect();+ let mut batches = stream::iter(batches)+ .map(|batch| async move {+ let texts: Vec<String> = batch.iter().map(|(_, t)| t.clone()).collect();+ let _permit = llm_util::acquire_embed(state, settings).await;+ let embeddings = client.embed(&texts).await?;+ if embeddings.len() != batch.len() {+ anyhow::bail!(+ "嵌入返回 {} 条,送去的是 {} 条",+ embeddings.len(),+ batch.len()+ );+ }+ let items: Vec<(Uuid, Vec<f32>)> =+ batch.iter().map(|(id, _)| *id).zip(embeddings).collect();+ utopia_store::documents::set_embeddings(&state.pool, &items).await?;+ Ok::<usize, anyhow::Error>(items.len())+ })+ .buffer_unordered(EMBED_JOBS);+ let mut done = 0;+ while let Some(written) = batches.next().await {+ done += written?;+ }+ Ok(done)+}++#[cfg(test)]+#[path = "pipeline_tests.rs"]+mod pipeline_tests;diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rsnew file mode 100644index 00000000..e9c6657d--- /dev/null+++ b/crates/utopia-server/src/pipeline_tests.rs@@ -0,0 +1,377 @@+//! #513:摄入的嵌入几批并发在飞,配对一条不乱。+//!+//! 并发不许碰的是「哪条向量落在哪个分块上」:错位落库之后看不出来,正文还在,向量是别人的。+//! 所以这里的测试问的是配对与完整,不是快慢。嵌入端点用 wiremock 假扮,每条正文的向量+//! 由正文本身算出来,读回来就能验;顺带记下每个请求的到达时刻和批大小。+//!+//! 1. **每个分块拿到自己正文的向量。** 40 条、三批、四路并发,批次乱序完成,读回来逐条对。+//! 附带:末尾不足一批的余数照样送,空文档一个请求都不发。+//! 2. **数量对不上整批放弃**:少回一条,那一批一条都不写。+//! 3. **上限守得住**:十二批、每批延时 150ms,同一时刻在飞的从不超过 EMBED_JOBS,+//! 总耗时明显短于串行。+//! 4. **一批失败文档不悬着**:走完整的 process_document,第二批回 500,文档落在 failed+//! 并带原因,不是停在 embedding。+//! 5. **就绪之前全部嵌完**:process_document 走通后没有一条向量为空。+//! 6. **记忆摄入走同一条路**:memory_ingest 嵌完自己的分块。+//!+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。++use std::sync::{Arc, Mutex};+use std::time::{Duration, Instant};+use utopia_core::models::Proposer;+use uuid::Uuid;+use wiremock::{+ matchers::method, matchers::path, Mock, MockServer, Request, Respond, ResponseTemplate,+};++/// 正文 → 向量:长度、首字符、字节和取模、常数 1。四维就够分辨每一条+fn vector_of(text: &str) -> Vec<f32> {+ vec![+ text.chars().count() as f32,+ text.chars().next().map(|c| c as u32 as f32).unwrap_or(0.0),+ (text.bytes().map(u32::from).sum::<u32>() % 97) as f32,+ 1.0,+ ]+}++/// 假嵌入端点。`short_by` 每批少回几条;`fail_request` 第几个请求回 500;`delay` 每个响应压多久。+/// 可克隆:一份挂进 wiremock,一份留在夹具里读到达记录+#[derive(Clone)]+struct FakeEmbed {+ arrivals: Arc<Mutex<Vec<(Instant, usize)>>>,+ delay: Duration,+ short_by: usize,+ fail_request: Option<usize>,+}++impl FakeEmbed {+ fn new(delay: Duration) -> Self {+ Self {+ arrivals: Arc::new(Mutex::new(Vec::new())),+ delay,+ short_by: 0,+ fail_request: None,+ }+ }+ fn arrivals(&self) -> Vec<(Instant, usize)> {+ self.arrivals.lock().expect("arrivals lock").clone()+ }+}++impl Respond for FakeEmbed {+ fn respond(&self, request: &Request) -> ResponseTemplate {+ let body: serde_json::Value = request.body_json().expect("embedding request is JSON");+ let inputs = body["input"].as_array().cloned().unwrap_or_default();+ let ordinal = {+ let mut a = self.arrivals.lock().expect("arrivals lock");+ a.push((Instant::now(), inputs.len()));+ a.len()+ };+ if self.fail_request == Some(ordinal) {+ return ResponseTemplate::new(500).set_body_string("model is down");+ }+ let data: Vec<serde_json::Value> = inputs+ .iter()+ .take(inputs.len().saturating_sub(self.short_by))+ .map(|t| serde_json::json!({ "embedding": vector_of(t.as_str().unwrap_or("")) }))+ .collect();+ ResponseTemplate::new(200)+ .set_body_json(serde_json::json!({ "data": data }))+ .set_delay(self.delay)+ }+}++struct Fx {+ pool: sqlx::PgPool,+ state: crate::state::AppState,+ server: MockServer,+ fake: FakeEmbed,+ org: Uuid,+ ws: Uuid,+ kb: Uuid,+ dir: std::path::PathBuf,+}++async fn fixture(fake: FakeEmbed) -> anyhow::Result<Option<Fx>> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(None);+ };+ let pool = sqlx::PgPool::connect(&url).await?;+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'pipeline-test')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'pipeline-test')")+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'pipeline-test')")+ .bind(kb)+ .bind(ws)+ .execute(&pool)+ .await?;+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/embeddings"))+ .respond_with(fake.clone())+ .mount(&server)+ .await;+ utopia_store::settings::upsert(+ &pool,+ ws,+ None,+ None,+ None,+ Some(&server.uri()),+ None,+ Some("fake-embed"),+ None,+ )+ .await?;+ let dir = std::env::temp_dir().join(format!("utopia-pipeline-{kb}"));+ let cfg = utopia_core::config::AppConfig {+ data_dir: dir.to_string_lossy().into_owned(),+ ..Default::default()+ };+ let search = Arc::new(utopia_search::SearchIndex::open(&dir.join("search"))?);+ let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into());+ Ok(Some(Fx {+ pool,+ state,+ server,+ fake,+ org,+ ws,+ kb,+ dir,+ }))+}++impl Fx {+ /// 一篇只有分块、没有向量的文档;正文各不相同,配对错了就对不上+ async fn document_with_chunks(&self, n: usize) -> anyhow::Result<Uuid> {+ let doc = Uuid::now_v7();+ sqlx::query(+ "INSERT INTO documents(id,kb_id,filename,sha256,status) VALUES($1,$2,'m.md',$3,'ready')",+ )+ .bind(doc)+ .bind(self.kb)+ .bind(format!("sha-{doc}"))+ .execute(&self.pool)+ .await?;+ for seq in 0..n {+ sqlx::query("INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES($1,$2,$3,$4,$5)")+ .bind(Uuid::now_v7())+ .bind(self.kb)+ .bind(doc)+ .bind(seq as i32)+ .bind(format!("chunk {seq} of {doc}: {}", "x".repeat(seq % 7)))+ .execute(&self.pool)+ .await?;+ }+ Ok(doc)+ }++ /// 一篇真文档:正文进 blob 存储,行是 pending,等 process_document 来解析分块+ async fn document_to_process(&self, paragraphs: usize) -> anyhow::Result<Uuid> {+ use sha2::{Digest, Sha256};+ let text: String = (0..paragraphs)+ .map(|i| format!("Paragraph {i}. {}\n\n", format!("word{i} ").repeat(140)))+ .collect();+ let sha: String = Sha256::digest(text.as_bytes())+ .iter()+ .map(|b| format!("{b:02x}"))+ .collect();+ self.state.blob.put(&sha, text.as_bytes()).await?;+ Ok(utopia_store::documents::create(+ &self.pool,+ self.kb,+ "long.md",+ "text/markdown",+ text.len() as i64,+ &sha,+ None,+ None,+ None,+ )+ .await?+ .id)+ }++ async fn embed(&self, doc: Uuid) -> anyhow::Result<usize> {+ let settings = utopia_store::settings::get(&self.pool, self.ws)+ .await?+ .expect("settings were written by the fixture");+ let client = crate::llm_util::embed_client(&settings).expect("embed model is configured");+ super::embed_pending(&self.state, &settings, &client, doc).await+ }++ /// (正文, 向量) 按 seq;向量为空就是没嵌+ async fn stored(&self, doc: Uuid) -> anyhow::Result<Vec<(String, Option<Vec<f32>>)>> {+ Ok(sqlx::query_as(+ "SELECT text, embedding::real[] FROM chunks WHERE document_id = $1 ORDER BY seq",+ )+ .bind(doc)+ .fetch_all(&self.pool)+ .await?)+ }++ async fn cleanup(self) -> anyhow::Result<()> {+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(self.org)+ .execute(&self.pool)+ .await?;+ let _ = std::fs::remove_dir_all(&self.dir);+ drop(self.server);+ Ok(())+ }+}++#[tokio::test]+async fn every_chunk_gets_the_vector_of_its_own_text() -> anyhow::Result<()> {+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(30))).await? else {+ return Ok(());+ };+ let doc = f.document_with_chunks(40).await?;+ assert_eq!(f.embed(doc).await?, 40);+ for (text, vector) in f.stored(doc).await? {+ assert_eq!(+ vector.as_deref(),+ Some(vector_of(&text).as_slice()),+ "chunk {text:?} must carry its own vector"+ );+ }+ let mut sizes: Vec<usize> = f.fake.arrivals().into_iter().map(|(_, n)| n).collect();+ sizes.sort_unstable();+ assert_eq!(sizes, vec![8, 16, 16], "two full batches and the remainder");++ let empty = f.document_with_chunks(0).await?;+ assert_eq!(f.embed(empty).await?, 0);+ assert_eq!(+ f.fake.arrivals().len(),+ 3,+ "an empty document makes no request"+ );+ f.cleanup().await+}++#[tokio::test]+async fn a_batch_that_answers_with_the_wrong_count_is_abandoned_whole() -> anyhow::Result<()> {+ let mut fake = FakeEmbed::new(Duration::ZERO);+ fake.short_by = 1;+ let Some(f) = fixture(fake).await? else {+ return Ok(());+ };+ let doc = f.document_with_chunks(10).await?;+ assert!(+ f.embed(doc).await.is_err(),+ "15 vectors for 16 texts is an error"+ );+ assert!(+ f.stored(doc).await?.iter().all(|(_, v)| v.is_none()),+ "nothing from a misaligned batch is written"+ );+ f.cleanup().await+}++#[tokio::test]+async fn the_embedding_gate_is_never_held_beyond_its_ceiling() -> anyhow::Result<()> {+ let delay = Duration::from_millis(150);+ let Some(f) = fixture(FakeEmbed::new(delay)).await? else {+ return Ok(());+ };+ let batches = 12;+ let doc = f.document_with_chunks(batches * super::EMBED_BATCH).await?;+ let started = Instant::now();+ assert_eq!(f.embed(doc).await?, batches * super::EMBED_BATCH);+ let elapsed = started.elapsed();++ // 在飞数 = 到达时刻落在同一个响应延时窗口里的请求数(窗口略短于延时,吃掉抖动)+ let arrivals = f.fake.arrivals();+ let window = delay - Duration::from_millis(20);+ let peak = arrivals+ .iter()+ .map(|(t, _)| {+ arrivals+ .iter()+ .filter(|(u, _)| *u <= *t && t.duration_since(*u) < window)+ .count()+ })+ .max()+ .unwrap_or(0);+ assert!(+ peak <= super::EMBED_JOBS,+ "peak in-flight batches {peak} exceeds the ceiling {}",+ super::EMBED_JOBS+ );+ assert!(peak >= 2, "batches actually overlap; peak was {peak}");+ let serial = delay * batches as u32;+ assert!(+ elapsed < serial / 2,+ "twelve batches took {elapsed:?}; serial would be {serial:?}"+ );+ f.cleanup().await+}++#[tokio::test]+async fn a_failed_batch_does_not_strand_the_document() -> anyhow::Result<()> {+ let mut fake = FakeEmbed::new(Duration::from_millis(20));+ fake.fail_request = Some(2);+ let Some(f) = fixture(fake).await? else {+ return Ok(());+ };+ let doc = f.document_to_process(20).await?;+ assert!(super::process_document(&f.state, doc).await.is_err());+ let row = utopia_store::documents::get(&f.pool, doc).await?;+ assert_eq!(row.status, "failed", "not left sitting in embedding");+ assert!(+ row.error.as_deref().is_some_and(|e| !e.is_empty()),+ "the reason is on the document"+ );+ f.cleanup().await+}++#[tokio::test]+async fn a_document_is_fully_embedded_before_it_is_ready() -> anyhow::Result<()> {+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(20))).await? else {+ return Ok(());+ };+ let doc = f.document_to_process(20).await?;+ super::process_document(&f.state, doc).await?;+ let row = utopia_store::documents::get(&f.pool, doc).await?;+ assert_eq!(row.status, "ready");+ let stored = f.stored(doc).await?;+ assert_eq!(stored.len() as i32, row.chunk_count);+ assert!(+ stored.len() > super::EMBED_BATCH,+ "enough chunks for more than one batch"+ );+ assert!(+ stored.iter().all(|(_, v)| v.is_some()),+ "no chunk is left without a vector when the document is ready"+ );+ f.cleanup().await+}++#[tokio::test]+async fn a_memory_episode_embeds_by_the_same_path_as_a_document() -> anyhow::Result<()> {+ let Some(f) = fixture(FakeEmbed::new(Duration::ZERO)).await? else {+ return Ok(());+ };+ let doc = f.document_with_chunks(5).await?;+ super::memory_ingest(+ &f.state,+ doc,+ Proposer {+ user_id: None,+ token_id: None,+ },+ )+ .await?;+ for (text, vector) in f.stored(doc).await? {+ assert_eq!(vector.as_deref(), Some(vector_of(&text).as_slice()));+ }+ f.cleanup().await+}