World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A NUL byte does not fail the whole document (#630)
WaylandYangc08118a975c8
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rsindex 590a0f30..87e023f6 100644--- a/crates/utopia-server/src/pipeline.rs+++ b/crates/utopia-server/src/pipeline.rs@@ -62,10 +62,13 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {let filename = doc.filename.clone();let parsed =tokio::task::spawn_blocking(move || utopia_ingest::parse(&filename, &bytes)).await??;- let text_len = parsed.text.chars().count() as i32;+ // 解析出来的正文可能夹着 NUL,入库之前剥掉(见 `without_nul`)。之后的长度、分块、+ // 全文索引、嵌入读的都是这一份,彼此的偏移才对得上+ let text = without_nul(&parsed.text);+ let text_len = text.chars().count() as i32;// 2. 分块 + 入库- let pieces = utopia_ingest::chunk_with_budget(&parsed.text, state.chunk_tokens);+ let pieces = utopia_ingest::chunk_with_budget(&text, state.chunk_tokens);let chunk_pairs =utopia_store::documents::replace_chunks(&state.pool, doc.kb_id, document_id, &pieces).await?;@@ -224,6 +227,26 @@ async fn embed_pending(Ok(done)}+/// 正文里的 NUL(`\0`)剥掉。+///+/// PDF 的文本层会夹带它——字体的 `ToUnicode` 映射与编码过的内容流是常见来源——而+/// Postgres 的 `TEXT` 不收 0x00:`chunks.text` 一插就报+/// `invalid byte sequence for encoding "UTF8": 0x00`,事务回滚,**整篇文档钉在 failed**,+/// 坏的只是其中几个字节。剥在这里,是因为文档正文进库只有这一条路(`replace_chunks`+/// 只有这一个调用方;记忆的 episode 另走 `memory::append_episode`,不经解析器),而且+/// 要在算长度和分块**之前**:放到更下游,存下的 `text_len` 与分块偏移就和真正入库的+/// 正文对不上了。+///+/// 绝大多数文档一个 NUL 都没有,那时原样借用,不为每篇文档复制一整份正文。+/// (#611,Jun Du / @plpycoin 报告并给出修法)+fn without_nul(text: &str) -> std::borrow::Cow<'_, str> {+ if text.contains('\0') {+ std::borrow::Cow::Owned(text.replace('\0', ""))+ } else {+ std::borrow::Cow::Borrowed(text)+ }+}+#[cfg(test)]#[path = "pipeline_tests.rs"]mod pipeline_tests;diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rsindex e9c6657d..fb3598f1 100644--- a/crates/utopia-server/src/pipeline_tests.rs+++ b/crates/utopia-server/src/pipeline_tests.rs@@ -13,6 +13,8 @@//! 并带原因,不是停在 embedding。//! 5. **就绪之前全部嵌完**:process_document 走通后没有一条向量为空。//! 6. **记忆摄入走同一条路**:memory_ingest 嵌完自己的分块。+//! 7. **正文夹 NUL 不毁整篇**(#611):Postgres 的 TEXT 不收 0x00,从前一个字节就让整篇+//! 落在 failed。现在走完整的 process_document 到 ready,库里没有一个分块带 NUL。//!//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。@@ -176,10 +178,15 @@ impl Fx {/// 一篇真文档:正文进 blob 存储,行是 pending,等 process_document 来解析分块async fn document_to_process(&self, paragraphs: usize) -> anyhow::Result<Uuid> {- use sha2::{Digest, Sha256};let text: String = (0..paragraphs).map(|i| format!("Paragraph {i}. {}\n\n", format!("word{i} ").repeat(140))).collect();+ self.document_with_text(&text).await+ }++ /// 同上,正文由调用方给+ async fn document_with_text(&self, text: &str) -> anyhow::Result<Uuid> {+ use sha2::{Digest, Sha256};let sha: String = Sha256::digest(text.as_bytes()).iter().map(|b| format!("{b:02x}"))@@ -333,6 +340,41 @@ async fn a_failed_batch_does_not_strand_the_document() -> anyhow::Result<()> {f.cleanup().await}+/// #611:一个 NUL 从前让整篇文档失败——坏的只是几个字节,丢的是整篇+#[tokio::test]+async fn a_nul_byte_does_not_fail_the_whole_document() -> anyhow::Result<()> {+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(5))).await? else {+ return Ok(());+ };+ // PDF 文本层里夹带 NUL 的样子:落在词中间、段落之间、一连好几个+ let text = "Revenue grew\0 twelve percent.\n\n\0\0Margins held at thirty\0-one.\n";+ let doc = f.document_with_text(text).await?;+ super::process_document(&f.state, doc).await?;++ let row = utopia_store::documents::get(&f.pool, doc).await?;+ assert_eq!(row.status, "ready", "a NUL byte must not fail the document");+ let stored = f.stored(doc).await?;+ assert!(!stored.is_empty(), "the document was chunked");+ assert!(+ stored.iter().all(|(t, _)| !t.contains('\0')),+ "no stored chunk carries a NUL"+ );+ let joined: String = stored.iter().map(|(t, _)| t.as_str()).collect();+ assert!(+ joined.contains("Revenue grew twelve percent."),+ "the words around the NUL survive, joined as written"+ );+ f.cleanup().await+}++#[test]+fn without_nul_borrows_when_there_is_nothing_to_strip() {+ use std::borrow::Cow;+ assert!(matches!(super::without_nul("plain text"), Cow::Borrowed(_)));+ assert_eq!(super::without_nul("a\0b\0\0c"), "abc");+ assert_eq!(super::without_nul("\0"), "");+}+#[tokio::test]async fn a_document_is_fully_embedded_before_it_is_ready() -> anyhow::Result<()> {let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(20))).await? else {