World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Move without_nul to utopia-core and strip NUL in memory episodes (#667)
Royceb71cf0f04690
diff --git a/crates/utopia-core/src/lib.rs b/crates/utopia-core/src/lib.rsindex 9296c035..70e46f47 100644--- a/crates/utopia-core/src/lib.rs+++ b/crates/utopia-core/src/lib.rs@@ -4,5 +4,7 @@ pub mod config;pub mod error;pub mod models;pub mod secrets;+pub mod text;pub use error::{is_deferred, is_terminal, AppError, AppResult, Deferred, Terminal};+pub use text::without_nul;diff --git a/crates/utopia-core/src/text.rs b/crates/utopia-core/src/text.rsnew file mode 100644index 00000000..eaa9d013--- /dev/null+++ b/crates/utopia-core/src/text.rs@@ -0,0 +1,37 @@+//! 进 `TEXT` 列之前的文本整理。+//!+//! Postgres 的 `TEXT` 不收 0x00:一插就报 `invalid byte sequence for encoding "UTF8": 0x00`,+//! 事务回滚,坏的只是几个字节,丢的是整篇文档或整条记忆。正文进 `chunks.text` 有两条路,+//! 两条都要剥:解析出来的文档正文(#611 / #630),和 `remember` 工具写下的记忆+//! (#665——JSON 里 U+0000 的转义是合法的,`serde_json` 解回来就是 NUL)。++use std::borrow::Cow;++/// 把 NUL(`\0`)剥掉。+///+/// 绝大多数文本一个 NUL 都没有,那时原样借用,不为每篇正文复制一整份。+pub fn without_nul(text: &str) -> Cow<'_, str> {+ if text.contains('\0') {+ Cow::Owned(text.replace('\0', ""))+ } else {+ Cow::Borrowed(text)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn borrows_when_there_is_nothing_to_strip() {+ assert!(matches!(without_nul("plain text"), Cow::Borrowed(_)));+ assert!(matches!(without_nul("中文正文"), Cow::Borrowed(_)));+ }++ #[test]+ fn strips_every_nul_and_keeps_everything_else() {+ assert_eq!(without_nul("a\0b\0\0c"), "abc");+ assert_eq!(without_nul("\0"), "");+ assert_eq!(without_nul("营收\0增长"), "营收增长");+ }+}diff --git a/crates/utopia-server/src/api/tools.rs b/crates/utopia-server/src/api/tools.rsindex 321cf442..a09e2093 100644--- a/crates/utopia-server/src/api/tools.rs+++ b/crates/utopia-server/src/api/tools.rs@@ -585,7 +585,11 @@ pub async fn query_data(ctx: &ToolCtx<'_>, args: &serde_json::Value) -> ToolResu}pub async fn remember(ctx: &ToolCtx<'_>, args: &serde_json::Value) -> ToolResult {- let text = args["text"].as_str().map(str::trim).unwrap_or("");+ // NUL 在入口就剥(#665):`append_episode` 自己也剥,但这里拼的回复与卡片详情+ // 用的是同一份文本——对话里它们要落进会话记录,Postgres 的 TEXT 与 JSONB 一样+ // 不收 0x00,一轮对话会因为回显了几个字节而存不下来+ let text = utopia_core::without_nul(args["text"].as_str().unwrap_or(""));+ let text = text.trim();// 与 `at` 同一种写法:YYYY / YYYY-MM / YYYY-MM-DD 或 RFC3339。日期落在那段第一天的// 正午——离两边的日界都最远;时刻照给的。回显按给的精度写,不把「2023 年」说成 1 月 1 日let (occurred_at, occurred_text) = match args["occurred_at"].as_str().map(str::trim) {diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rsindex 87e023f6..75bfd053 100644--- a/crates/utopia-server/src/pipeline.rs+++ b/crates/utopia-server/src/pipeline.rs@@ -62,9 +62,10 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {let filename = doc.filename.clone();let parsed =tokio::task::spawn_blocking(move || utopia_ingest::parse(&filename, &bytes)).await??;- // 解析出来的正文可能夹着 NUL,入库之前剥掉(见 `without_nul`)。之后的长度、分块、- // 全文索引、嵌入读的都是这一份,彼此的偏移才对得上- let text = without_nul(&parsed.text);+ // 解析出来的正文可能夹着 NUL(PDF 文本层常见),入库之前剥掉(#611)——与记忆+ // 那条路共用 `utopia_core::without_nul`(#665)。剥必须在算长度、分块之前:之后的+ // text_len、分块偏移、全文索引、嵌入读的都是这一份,彼此才对得上+ let text = utopia_core::without_nul(&parsed.text);let text_len = text.chars().count() as i32;// 2. 分块 + 入库@@ -227,26 +228,6 @@ async fn embed_pending(Ok(done)}-/// 正文里的 NUL(`\0`)剥掉。-///-/// PDF 的文本层会夹带它——字体的 `ToUnicode` 映射与编码过的内容流是常见来源——而-/// Postgres 的 `TEXT` 不收 0x00:`chunks.text` 一插就报-/// `invalid byte sequence for encoding "UTF8": 0x00`,事务回滚,**整篇文档钉在 failed**,-/// 坏的只是其中几个字节。剥在这里,是因为文档正文进库只有这一条路(`replace_chunks`-/// 只有这一个调用方;记忆的 episode 另走 `memory::append_episode`,不经解析器),而且-/// 要在算长度和分块**之前**:放到更下游,存下的 `text_len` 与分块偏移就和真正入库的-/// 正文对不上了。-///-/// 绝大多数文档一个 NUL 都没有,那时原样借用,不为每篇文档复制一整份正文。-/// (#611,Jun Du / @plpycoin 报告并给出修法)-fn without_nul(text: &str) -> std::borrow::Cow<'_, str> {- if text.contains('\0') {- std::borrow::Cow::Owned(text.replace('\0', ""))- } else {- std::borrow::Cow::Borrowed(text)- }-}-#[cfg(test)]#[path = "pipeline_tests.rs"]mod pipeline_tests;diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rsindex fb3598f1..333a13c5 100644--- a/crates/utopia-server/src/pipeline_tests.rs+++ b/crates/utopia-server/src/pipeline_tests.rs@@ -370,9 +370,12 @@ async fn a_nul_byte_does_not_fail_the_whole_document() -> anyhow::Result<()> {#[test]fn without_nul_borrows_when_there_is_nothing_to_strip() {use std::borrow::Cow;- assert!(matches!(super::without_nul("plain text"), Cow::Borrowed(_)));- assert_eq!(super::without_nul("a\0b\0\0c"), "abc");- assert_eq!(super::without_nul("\0"), "");+ assert!(matches!(+ utopia_core::without_nul("plain text"),+ Cow::Borrowed(_)+ ));+ assert_eq!(utopia_core::without_nul("a\0b\0\0c"), "abc");+ assert_eq!(utopia_core::without_nul("\0"), "");}#[tokio::test]diff --git a/crates/utopia-store/src/memory.rs b/crates/utopia-store/src/memory.rsindex eb2fcd32..59a1e0b6 100644--- a/crates/utopia-store/src/memory.rs+++ b/crates/utopia-store/src/memory.rs@@ -73,6 +73,11 @@ pub async fn get_or_create_memory_doc(pool: &PgPool, kb_id: Uuid) -> AppResult<U/// 追加一条 episode:新 chunk(extracted_at 空 → 增量抽取会拾起;embedding 空 →/// memory_ingest 会补)。事发时间内嵌进文本首行,抽取模型据此定 valid_from。+///+/// **NUL 字节在入库前剥掉**(#665):chat / MCP 来的 `remember` 工具里 JSON+/// 解码出来的 `\0` 走到这一行——Postgres `TEXT` 不收 0x00,与文档路径同一+/// 个错误,整段就丢了。共用 `utopia_core::without_nul`,与+/// `pipeline::process_document` 走同一个 helper。pub async fn append_episode(pool: &PgPool,kb_id: Uuid,@@ -80,7 +85,13 @@ pub async fn append_episode(occurred_at: DateTime<Utc>,) -> AppResult<(Uuid, Uuid)> {let doc_id = get_or_create_memory_doc(pool, kb_id).await?;- let stamped = format!("[{}] {}", occurred_at.format("%Y-%m-%d %H:%M"), text.trim());+ // 先剥 NUL 再拼前缀:下面入库的 text 与 char_end 都从 `stamped` 来,算的是同一份+ let stripped = utopia_core::without_nul(text);+ let stamped = format!(+ "[{}] {}",+ occurred_at.format("%Y-%m-%d %H:%M"),+ stripped.trim()+ );let chunk_id = Uuid::now_v7();let mut tx = pool.begin().await?;sqlx::query(diff --git a/crates/utopia-store/tests/a_remembered_episode_strips_nul.rs b/crates/utopia-store/tests/a_remembered_episode_strips_nul.rsnew file mode 100644index 00000000..8a57b02e--- /dev/null+++ b/crates/utopia-store/tests/a_remembered_episode_strips_nul.rs@@ -0,0 +1,71 @@+//! `remember` 工具把一句记忆落成 chunk 之前先剥 NUL 字节(#665)。+//!+//! 与 `pipeline::process_document` 共用 `utopia_core::without_nul`:两份+//! 同一函数,所以测一处。记忆这条路径值得独立测一遍——+//! 它不经 `utopia_ingest::parse`,`pipeline` 那边的 NUL 测试覆盖不到。+//!+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。++use sqlx::PgPool;+use uuid::Uuid;++#[tokio::test]+async fn a_nul_in_a_remembered_episode_is_stripped_before_chunks_text() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ for (q, vals) in [+ (+ "INSERT INTO organizations (id, name) VALUES ($1, 'remember-nul-test')",+ vec![org],+ ),+ (+ "INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'remember-nul-test')",+ vec![ws, org],+ ),+ (+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'remember-nul-test')",+ vec![kb, ws],+ ),+ ] {+ let mut stmt = sqlx::query(q);+ for v in vals {+ stmt = stmt.bind(v);+ }+ stmt.execute(&pool).await?;+ }++ // 三种 NUL:词里、词与词之间、连续 NUL。`serde_json` 把 JSON 的 \u0000 解回 `\0`,+ // 模型 / MCP 写回路径就是这一路。+ let raw = "Acme\0moved\0its\0\0HQ to Shenzhen on 2026-03-15.";+ let (_doc, chunk) =+ utopia_store::memory::append_episode(&pool, kb, raw, chrono::Utc::now()).await?;++ let (text, char_end): (String, i32) =+ sqlx::query_as("SELECT text, char_end FROM chunks WHERE id = $1")+ .bind(chunk)+ .fetch_one(&pool)+ .await?;++ assert!(!text.contains('\0'), "入库后还剩 NUL:{text:?}");+ // `[YYYY-MM-DD HH:MM] ` 前缀按当时的时间生成;断言只验**它之后**的内容:+ // 词被剥完后该照原样拼回去,NUL 整段消失、词与词之间的 NUL 也消失。+ let stripped = text.split_once("] ").map(|(_, rest)| rest).unwrap_or("");+ assert_eq!(+ stripped, "AcmemoveditsHQ to Shenzhen on 2026-03-15.",+ "剥完 NUL 之后词该照原来拼回去"+ );+ assert_eq!(+ char_end as usize,+ text.chars().count(),+ "char_end 用的是剥完的那份,不是入库前的那份"+ );++ sqlx::query("DELETE FROM organizations WHERE id = $1")+ .bind(org)+ .execute(&pool)+ .await?;+ Ok(())+}