Skip to content

World's first open-source enterprise world model.

Move without_nul to utopia-core and strip NUL in memory episodes (#667)

Royceb71cf0f04690
diff --git a/crates/utopia-core/src/lib.rs b/crates/utopia-core/src/lib.rs
index 9296c035..70e46f47 100644
--- a/crates/utopia-core/src/lib.rs
+++ b/crates/utopia-core/src/lib.rs
@@ -4,5 +4,7 @@ pub mod config;
pub mod error;
pub mod models;
pub mod secrets;
+pub mod text;
pub use error::{is_deferred, is_terminal, AppError, AppResult, Deferred, Terminal};
+pub use text::without_nul;
diff --git a/crates/utopia-core/src/text.rs b/crates/utopia-core/src/text.rs
new file mode 100644
index 00000000..eaa9d013
--- /dev/null
+++ b/crates/utopia-core/src/text.rs
@@ -0,0 +1,37 @@
+//! 进 `TEXT` 列之前的文本整理。
+//!
+//! Postgres 的 `TEXT` 不收 0x00:一插就报 `invalid byte sequence for encoding "UTF8": 0x00`,
+//! 事务回滚,坏的只是几个字节,丢的是整篇文档或整条记忆。正文进 `chunks.text` 有两条路,
+//! 两条都要剥:解析出来的文档正文(#611 / #630),和 `remember` 工具写下的记忆
+//! (#665——JSON 里 U+0000 的转义是合法的,`serde_json` 解回来就是 NUL)。
+
+use std::borrow::Cow;
+
+/// 把 NUL(`\0`)剥掉。
+///
+/// 绝大多数文本一个 NUL 都没有,那时原样借用,不为每篇正文复制一整份。
+pub fn without_nul(text: &str) -> Cow<'_, str> {
+ if text.contains('\0') {
+ Cow::Owned(text.replace('\0', ""))
+ } else {
+ Cow::Borrowed(text)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn borrows_when_there_is_nothing_to_strip() {
+ assert!(matches!(without_nul("plain text"), Cow::Borrowed(_)));
+ assert!(matches!(without_nul("中文正文"), Cow::Borrowed(_)));
+ }
+
+ #[test]
+ fn strips_every_nul_and_keeps_everything_else() {
+ assert_eq!(without_nul("a\0b\0\0c"), "abc");
+ assert_eq!(without_nul("\0"), "");
+ assert_eq!(without_nul("营收\0增长"), "营收增长");
+ }
+}
diff --git a/crates/utopia-server/src/api/tools.rs b/crates/utopia-server/src/api/tools.rs
index 321cf442..a09e2093 100644
--- a/crates/utopia-server/src/api/tools.rs
+++ b/crates/utopia-server/src/api/tools.rs
@@ -585,7 +585,11 @@ pub async fn query_data(ctx: &ToolCtx<'_>, args: &serde_json::Value) -> ToolResu
}
pub async fn remember(ctx: &ToolCtx<'_>, args: &serde_json::Value) -> ToolResult {
- let text = args["text"].as_str().map(str::trim).unwrap_or("");
+ // NUL 在入口就剥(#665):`append_episode` 自己也剥,但这里拼的回复与卡片详情
+ // 用的是同一份文本——对话里它们要落进会话记录,Postgres 的 TEXT 与 JSONB 一样
+ // 不收 0x00,一轮对话会因为回显了几个字节而存不下来
+ let text = utopia_core::without_nul(args["text"].as_str().unwrap_or(""));
+ let text = text.trim();
// 与 `at` 同一种写法:YYYY / YYYY-MM / YYYY-MM-DD 或 RFC3339。日期落在那段第一天的
// 正午——离两边的日界都最远;时刻照给的。回显按给的精度写,不把「2023 年」说成 1 月 1 日
let (occurred_at, occurred_text) = match args["occurred_at"].as_str().map(str::trim) {
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rs
index 87e023f6..75bfd053 100644
--- a/crates/utopia-server/src/pipeline.rs
+++ b/crates/utopia-server/src/pipeline.rs
@@ -62,9 +62,10 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
let filename = doc.filename.clone();
let parsed =
tokio::task::spawn_blocking(move || utopia_ingest::parse(&filename, &bytes)).await??;
- // 解析出来的正文可能夹着 NUL,入库之前剥掉(见 `without_nul`)。之后的长度、分块、
- // 全文索引、嵌入读的都是这一份,彼此的偏移才对得上
- let text = without_nul(&parsed.text);
+ // 解析出来的正文可能夹着 NUL(PDF 文本层常见),入库之前剥掉(#611)——与记忆
+ // 那条路共用 `utopia_core::without_nul`(#665)。剥必须在算长度、分块之前:之后的
+ // text_len、分块偏移、全文索引、嵌入读的都是这一份,彼此才对得上
+ let text = utopia_core::without_nul(&parsed.text);
let text_len = text.chars().count() as i32;
// 2. 分块 + 入库
@@ -227,26 +228,6 @@ async fn embed_pending(
Ok(done)
}
-/// 正文里的 NUL(`\0`)剥掉。
-///
-/// PDF 的文本层会夹带它——字体的 `ToUnicode` 映射与编码过的内容流是常见来源——而
-/// Postgres 的 `TEXT` 不收 0x00:`chunks.text` 一插就报
-/// `invalid byte sequence for encoding "UTF8": 0x00`,事务回滚,**整篇文档钉在 failed**,
-/// 坏的只是其中几个字节。剥在这里,是因为文档正文进库只有这一条路(`replace_chunks`
-/// 只有这一个调用方;记忆的 episode 另走 `memory::append_episode`,不经解析器),而且
-/// 要在算长度和分块**之前**:放到更下游,存下的 `text_len` 与分块偏移就和真正入库的
-/// 正文对不上了。
-///
-/// 绝大多数文档一个 NUL 都没有,那时原样借用,不为每篇文档复制一整份正文。
-/// (#611,Jun Du / @plpycoin 报告并给出修法)
-fn without_nul(text: &str) -> std::borrow::Cow<'_, str> {
- if text.contains('\0') {
- std::borrow::Cow::Owned(text.replace('\0', ""))
- } else {
- std::borrow::Cow::Borrowed(text)
- }
-}
-
#[cfg(test)]
#[path = "pipeline_tests.rs"]
mod pipeline_tests;
diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rs
index fb3598f1..333a13c5 100644
--- a/crates/utopia-server/src/pipeline_tests.rs
+++ b/crates/utopia-server/src/pipeline_tests.rs
@@ -370,9 +370,12 @@ async fn a_nul_byte_does_not_fail_the_whole_document() -> anyhow::Result<()> {
#[test]
fn without_nul_borrows_when_there_is_nothing_to_strip() {
use std::borrow::Cow;
- assert!(matches!(super::without_nul("plain text"), Cow::Borrowed(_)));
- assert_eq!(super::without_nul("a\0b\0\0c"), "abc");
- assert_eq!(super::without_nul("\0"), "");
+ assert!(matches!(
+ utopia_core::without_nul("plain text"),
+ Cow::Borrowed(_)
+ ));
+ assert_eq!(utopia_core::without_nul("a\0b\0\0c"), "abc");
+ assert_eq!(utopia_core::without_nul("\0"), "");
}
#[tokio::test]
diff --git a/crates/utopia-store/src/memory.rs b/crates/utopia-store/src/memory.rs
index eb2fcd32..59a1e0b6 100644
--- a/crates/utopia-store/src/memory.rs
+++ b/crates/utopia-store/src/memory.rs
@@ -73,6 +73,11 @@ pub async fn get_or_create_memory_doc(pool: &PgPool, kb_id: Uuid) -> AppResult<U
/// 追加一条 episode:新 chunk(extracted_at 空 → 增量抽取会拾起;embedding 空 →
/// memory_ingest 会补)。事发时间内嵌进文本首行,抽取模型据此定 valid_from。
+///
+/// **NUL 字节在入库前剥掉**(#665):chat / MCP 来的 `remember` 工具里 JSON
+/// 解码出来的 `\0` 走到这一行——Postgres `TEXT` 不收 0x00,与文档路径同一
+/// 个错误,整段就丢了。共用 `utopia_core::without_nul`,与
+/// `pipeline::process_document` 走同一个 helper。
pub async fn append_episode(
pool: &PgPool,
kb_id: Uuid,
@@ -80,7 +85,13 @@ pub async fn append_episode(
occurred_at: DateTime<Utc>,
) -> AppResult<(Uuid, Uuid)> {
let doc_id = get_or_create_memory_doc(pool, kb_id).await?;
- let stamped = format!("[{}] {}", occurred_at.format("%Y-%m-%d %H:%M"), text.trim());
+ // 先剥 NUL 再拼前缀:下面入库的 text 与 char_end 都从 `stamped` 来,算的是同一份
+ let stripped = utopia_core::without_nul(text);
+ let stamped = format!(
+ "[{}] {}",
+ occurred_at.format("%Y-%m-%d %H:%M"),
+ stripped.trim()
+ );
let chunk_id = Uuid::now_v7();
let mut tx = pool.begin().await?;
sqlx::query(
diff --git a/crates/utopia-store/tests/a_remembered_episode_strips_nul.rs b/crates/utopia-store/tests/a_remembered_episode_strips_nul.rs
new file mode 100644
index 00000000..8a57b02e
--- /dev/null
+++ b/crates/utopia-store/tests/a_remembered_episode_strips_nul.rs
@@ -0,0 +1,71 @@
+//! `remember` 工具把一句记忆落成 chunk 之前先剥 NUL 字节(#665)。
+//!
+//! 与 `pipeline::process_document` 共用 `utopia_core::without_nul`:两份
+//! 同一函数,所以测一处。记忆这条路径值得独立测一遍——
+//! 它不经 `utopia_ingest::parse`,`pipeline` 那边的 NUL 测试覆盖不到。
+//!
+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
+
+use sqlx::PgPool;
+use uuid::Uuid;
+
+#[tokio::test]
+async fn a_nul_in_a_remembered_episode_is_stripped_before_chunks_text() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ for (q, vals) in [
+ (
+ "INSERT INTO organizations (id, name) VALUES ($1, 'remember-nul-test')",
+ vec![org],
+ ),
+ (
+ "INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'remember-nul-test')",
+ vec![ws, org],
+ ),
+ (
+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'remember-nul-test')",
+ vec![kb, ws],
+ ),
+ ] {
+ let mut stmt = sqlx::query(q);
+ for v in vals {
+ stmt = stmt.bind(v);
+ }
+ stmt.execute(&pool).await?;
+ }
+
+ // 三种 NUL:词里、词与词之间、连续 NUL。`serde_json` 把 JSON 的 \u0000 解回 `\0`,
+ // 模型 / MCP 写回路径就是这一路。
+ let raw = "Acme\0moved\0its\0\0HQ to Shenzhen on 2026-03-15.";
+ let (_doc, chunk) =
+ utopia_store::memory::append_episode(&pool, kb, raw, chrono::Utc::now()).await?;
+
+ let (text, char_end): (String, i32) =
+ sqlx::query_as("SELECT text, char_end FROM chunks WHERE id = $1")
+ .bind(chunk)
+ .fetch_one(&pool)
+ .await?;
+
+ assert!(!text.contains('\0'), "入库后还剩 NUL:{text:?}");
+ // `[YYYY-MM-DD HH:MM] ` 前缀按当时的时间生成;断言只验**它之后**的内容:
+ // 词被剥完后该照原样拼回去,NUL 整段消失、词与词之间的 NUL 也消失。
+ let stripped = text.split_once("] ").map(|(_, rest)| rest).unwrap_or("");
+ assert_eq!(
+ stripped, "AcmemoveditsHQ to Shenzhen on 2026-03-15.",
+ "剥完 NUL 之后词该照原来拼回去"
+ );
+ assert_eq!(
+ char_end as usize,
+ text.chars().count(),
+ "char_end 用的是剥完的那份,不是入库前的那份"
+ );
+
+ sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ Ok(())
+}