Skip to content

World's first open-source enterprise world model.

A NUL byte does not fail the whole document (#630)

WaylandYangc08118a975c8
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rs
index 590a0f30..87e023f6 100644
--- a/crates/utopia-server/src/pipeline.rs
+++ b/crates/utopia-server/src/pipeline.rs
@@ -62,10 +62,13 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
let filename = doc.filename.clone();
let parsed =
tokio::task::spawn_blocking(move || utopia_ingest::parse(&filename, &bytes)).await??;
- let text_len = parsed.text.chars().count() as i32;
+ // 解析出来的正文可能夹着 NUL,入库之前剥掉(见 `without_nul`)。之后的长度、分块、
+ // 全文索引、嵌入读的都是这一份,彼此的偏移才对得上
+ let text = without_nul(&parsed.text);
+ let text_len = text.chars().count() as i32;
// 2. 分块 + 入库
- let pieces = utopia_ingest::chunk_with_budget(&parsed.text, state.chunk_tokens);
+ let pieces = utopia_ingest::chunk_with_budget(&text, state.chunk_tokens);
let chunk_pairs =
utopia_store::documents::replace_chunks(&state.pool, doc.kb_id, document_id, &pieces)
.await?;
@@ -224,6 +227,26 @@ async fn embed_pending(
Ok(done)
}
+/// 正文里的 NUL(`\0`)剥掉。
+///
+/// PDF 的文本层会夹带它——字体的 `ToUnicode` 映射与编码过的内容流是常见来源——而
+/// Postgres 的 `TEXT` 不收 0x00:`chunks.text` 一插就报
+/// `invalid byte sequence for encoding "UTF8": 0x00`,事务回滚,**整篇文档钉在 failed**,
+/// 坏的只是其中几个字节。剥在这里,是因为文档正文进库只有这一条路(`replace_chunks`
+/// 只有这一个调用方;记忆的 episode 另走 `memory::append_episode`,不经解析器),而且
+/// 要在算长度和分块**之前**:放到更下游,存下的 `text_len` 与分块偏移就和真正入库的
+/// 正文对不上了。
+///
+/// 绝大多数文档一个 NUL 都没有,那时原样借用,不为每篇文档复制一整份正文。
+/// (#611,Jun Du / @plpycoin 报告并给出修法)
+fn without_nul(text: &str) -> std::borrow::Cow<'_, str> {
+ if text.contains('\0') {
+ std::borrow::Cow::Owned(text.replace('\0', ""))
+ } else {
+ std::borrow::Cow::Borrowed(text)
+ }
+}
+
#[cfg(test)]
#[path = "pipeline_tests.rs"]
mod pipeline_tests;
diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rs
index e9c6657d..fb3598f1 100644
--- a/crates/utopia-server/src/pipeline_tests.rs
+++ b/crates/utopia-server/src/pipeline_tests.rs
@@ -13,6 +13,8 @@
//! 并带原因,不是停在 embedding。
//! 5. **就绪之前全部嵌完**:process_document 走通后没有一条向量为空。
//! 6. **记忆摄入走同一条路**:memory_ingest 嵌完自己的分块。
+//! 7. **正文夹 NUL 不毁整篇**(#611):Postgres 的 TEXT 不收 0x00,从前一个字节就让整篇
+//! 落在 failed。现在走完整的 process_document 到 ready,库里没有一个分块带 NUL。
//!
//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
@@ -176,10 +178,15 @@ impl Fx {
/// 一篇真文档:正文进 blob 存储,行是 pending,等 process_document 来解析分块
async fn document_to_process(&self, paragraphs: usize) -> anyhow::Result<Uuid> {
- use sha2::{Digest, Sha256};
let text: String = (0..paragraphs)
.map(|i| format!("Paragraph {i}. {}\n\n", format!("word{i} ").repeat(140)))
.collect();
+ self.document_with_text(&text).await
+ }
+
+ /// 同上,正文由调用方给
+ async fn document_with_text(&self, text: &str) -> anyhow::Result<Uuid> {
+ use sha2::{Digest, Sha256};
let sha: String = Sha256::digest(text.as_bytes())
.iter()
.map(|b| format!("{b:02x}"))
@@ -333,6 +340,41 @@ async fn a_failed_batch_does_not_strand_the_document() -> anyhow::Result<()> {
f.cleanup().await
}
+/// #611:一个 NUL 从前让整篇文档失败——坏的只是几个字节,丢的是整篇
+#[tokio::test]
+async fn a_nul_byte_does_not_fail_the_whole_document() -> anyhow::Result<()> {
+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(5))).await? else {
+ return Ok(());
+ };
+ // PDF 文本层里夹带 NUL 的样子:落在词中间、段落之间、一连好几个
+ let text = "Revenue grew\0 twelve percent.\n\n\0\0Margins held at thirty\0-one.\n";
+ let doc = f.document_with_text(text).await?;
+ super::process_document(&f.state, doc).await?;
+
+ let row = utopia_store::documents::get(&f.pool, doc).await?;
+ assert_eq!(row.status, "ready", "a NUL byte must not fail the document");
+ let stored = f.stored(doc).await?;
+ assert!(!stored.is_empty(), "the document was chunked");
+ assert!(
+ stored.iter().all(|(t, _)| !t.contains('\0')),
+ "no stored chunk carries a NUL"
+ );
+ let joined: String = stored.iter().map(|(t, _)| t.as_str()).collect();
+ assert!(
+ joined.contains("Revenue grew twelve percent."),
+ "the words around the NUL survive, joined as written"
+ );
+ f.cleanup().await
+}
+
+#[test]
+fn without_nul_borrows_when_there_is_nothing_to_strip() {
+ use std::borrow::Cow;
+ assert!(matches!(super::without_nul("plain text"), Cow::Borrowed(_)));
+ assert_eq!(super::without_nul("a\0b\0\0c"), "abc");
+ assert_eq!(super::without_nul("\0"), "");
+}
+
#[tokio::test]
async fn a_document_is_fully_embedded_before_it_is_ready() -> anyhow::Result<()> {
let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(20))).await? else {