World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Use opening datelines as document dates for text uploads (#645)
Daniel3bdabee59707
diff --git a/crates/utopia-server/src/api/documents_routes.rs b/crates/utopia-server/src/api/documents_routes.rsindex d3e311c3..bf111f71 100644--- a/crates/utopia-server/src/api/documents_routes.rs+++ b/crates/utopia-server/src/api/documents_routes.rs@@ -72,7 +72,7 @@ pub async fn upload(.await.map_err(AppError::Other)?;- match utopia_store::documents::create(+ match utopia_store::documents::create_from_upload(&state.pool,kb_id,&filename,@@ -80,8 +80,7 @@ pub async fn upload(bytes.len() as i64,&sha256,target_source,- None,- None,+ content_time(&filename, &bytes),).await{@@ -372,6 +371,49 @@ fn hex(bytes: &[u8]) -> String {bytes.iter().map(|b| format!("{b:02x}")).collect()}+/// 只认开头独立的完整日期行,不把正文里提到的事件日期当作文档日期(#610)。+fn content_time(filename: &str, bytes: &[u8]) -> Option<chrono::DateTime<chrono::Utc>> {+ let extension = std::path::Path::new(filename).extension()?.to_str()?;+ if !["txt", "md", "markdown"]+ .iter()+ .any(|ext| extension.eq_ignore_ascii_case(ext))+ {+ return None;+ }+ // 只解码头部 4 KiB:日期行只认开头。PDF、Word 这类格式要读日期时,在各自的解析器里+ // 读它们自己的元数据,不在这里猜+ const HEADER_BYTES: usize = 4096;+ let text = utopia_ingest::decode_text(&bytes[..bytes.len().min(HEADER_BYTES)]);+ let header = if bytes.len() > HEADER_BYTES {+ // 截断的半行可能在日期后还有文字,不能把它误当独立日期行。+ text.rsplit_once('\n')?.0+ } else {+ &text+ };+ let line = header+ .trim_start_matches('\u{feff}')+ .lines()+ .map(str::trim)+ .find(|line| !line.is_empty())?;+ let date = line+ .strip_prefix('(')+ .and_then(|s| s.strip_suffix(')'))+ .or_else(|| line.strip_prefix('(').and_then(|s| s.strip_suffix(')')))+ .unwrap_or(line);+ if !date.get(..4)?.bytes().all(|b| b.is_ascii_digit()) {+ return None;+ }+ let day = chrono::NaiveDate::parse_from_str(date, "%Y-%m-%d")+ .or_else(|_| chrono::NaiveDate::parse_from_str(date, "%Y年%m月%d日"))+ .ok()?;+ // 与项目已有日精度约定一致:UTC 零点是存储约定,不猜作者所在时区。+ Some(day.and_hms_opt(0, 0, 0)?.and_utc())+}++#[cfg(test)]+#[path = "documents_routes_tests.rs"]+mod tests;+/// 抽取丢弃信号:哪些事实抽出来了却没能落地。整库一次取回——按/// (文档 × 原因 × 具体对象) 聚合后行数很小,Library 既算总数又展开详情,/// 不必逐行发请求。diff --git a/crates/utopia-server/src/api/documents_routes_tests.rs b/crates/utopia-server/src/api/documents_routes_tests.rsnew file mode 100644index 00000000..d58a2c46--- /dev/null+++ b/crates/utopia-server/src/api/documents_routes_tests.rs@@ -0,0 +1,417 @@+use super::content_time;+use axum::body::{to_bytes, Body};+use axum::http::{Request, StatusCode};+use chrono::{DateTime, Utc};+use serde_json::{json, Value};+use std::sync::Arc;+use tower::ServiceExt;+use utopia_core::models::Document;+use utopia_store::documents;+use uuid::Uuid;++#[test]+fn only_a_complete_opening_dateline_sets_the_date() {+ let expected = "2024-02-29T00:00:00Z".parse::<DateTime<Utc>>().unwrap();+ for text in [+ "2024-02-29\nAnnouncement",+ "\u{feff}\r\n 2024-02-29 \r\nAnnouncement",+ "(2024年2月29日)\n公告",+ "(2024-02-29)",+ ] {+ for filename in ["filing.txt", "filing.MD", "filing.markdown"] {+ assert_eq!(content_time(filename, text.as_bytes()), Some(expected));+ }+ }+ for text in [+ "",+ " \n\t",+ "2023-02-29",+ "2024-13-01",+ "2024-02",+ "02/03/2024",+ "24-02-29",+ "2024-02-29 / 2025-02-28",+ "2024-02-29 was the previous meeting",+ "Announcement\n2024-02-29",+ "# 2024-02-29 announcement",+ "2024-02-29T12:00:00+08:00",+ "(2024年2月29日",+ ] {+ assert_eq!(content_time("filing.md", text.as_bytes()), None, "{text:?}");+ }+ for filename in [+ "filing.pdf",+ "filing.docx",+ "filing.csv",+ "2024-02-29",+ "filing",+ ] {+ assert_eq!(content_time(filename, b"2024-02-29"), None);+ }+}++#[test]+fn header_decoding_is_bounded_and_never_accepts_a_truncated_line() {+ let expected = "2024-02-29T00:00:00Z".parse::<DateTime<Utc>>().unwrap();+ let mut utf16 = vec![0xff, 0xfe];+ for unit in "(2024年2月29日)\r\n公告".encode_utf16() {+ utf16.extend(unit.to_le_bytes());+ }+ assert_eq!(content_time("filing.txt", &utf16), Some(expected));++ let mut long = b"2024-02-29\n".to_vec();+ long.extend("正文".repeat(2000).as_bytes());+ assert_eq!(content_time("filing.md", &long), Some(expected));+ let truncated = format!(+ "{}2024-02-29 is an event, not a dateline",+ "\n".repeat(4086)+ );+ assert_eq!(content_time("filing.txt", truncated.as_bytes()), None);+ let too_late = format!("{}2024-02-29\n", "\n".repeat(4096));+ assert_eq!(content_time("filing.txt", too_late.as_bytes()), None);+}++struct Fixture {+ pool: sqlx::PgPool,+ state: crate::state::AppState,+ app: axum::Router,+ org: Uuid,+ kb: Uuid,+ other_kb: Uuid,+ user: Uuid,+ folder: Uuid,+ foreign_folder: Uuid,+ source: Uuid,+ token: String,+ _dir: tempfile::TempDir,+}++impl Fixture {+ async fn new() -> anyhow::Result<Option<Self>> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(None);+ };+ let pool = sqlx::PgPool::connect(&url).await?;+ utopia_store::db::migrate(&pool).await?;+ let (org, ws, kb, other_kb, user, folder, foreign_folder, source) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ // Only locally generated UUIDs are interpolated into fixture SQL.+ sqlx::raw_sql(&format!(+ "INSERT INTO organizations(id,name) VALUES ('{org}','upload-date-test');+ INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','upload-date-test');+ INSERT INTO users(id,org_id,email,display_name,password_hash)+ VALUES ('{user}','{org}','{user}@upload.test','upload-test','unused');+ INSERT INTO knowledge_bases(id,workspace_id,name) VALUES+ ('{kb}','{ws}','uploads'), ('{other_kb}','{ws}','other-uploads');+ INSERT INTO kb_members(kb_id,user_id,role) VALUES+ ('{kb}','{user}','editor'), ('{other_kb}','{user}','editor');+ INSERT INTO sources(id,kb_id,kind,name) VALUES+ ('{folder}','{kb}','folder','uploads'),+ ('{foreign_folder}','{other_kb}','folder','other-folder'),+ ('{source}','{kb}','api','api-source');"+ ))+ .execute(&pool)+ .await?;+ // Processing may queue extraction, but no test calls a live model.+ utopia_store::settings::upsert(+ &pool,+ ws,+ Some("http://127.0.0.1:9"),+ None,+ Some("test-model"),+ None,+ None,+ None,+ None,+ )+ .await?;+ let dir = tempfile::tempdir()?;+ let cfg = utopia_core::config::AppConfig {+ data_dir: dir.path().to_string_lossy().into_owned(),+ ..Default::default()+ };+ let search = Arc::new(utopia_search::SearchIndex::open(+ &dir.path().join("search"),+ )?);+ let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into());+ let token = crate::auth::issue_token(&state, user)?;+ let app = super::super::router(state.clone(), &cfg);+ Ok(Some(Self {+ pool,+ state,+ app,+ org,+ kb,+ other_kb,+ user,+ folder,+ foreign_folder,+ source,+ token,+ _dir: dir,+ }))+ }++ async fn upload(+ &self,+ kb: Uuid,+ query: &str,+ files: &[(&str, &str)],+ ) -> anyhow::Result<(StatusCode, Value)> {+ let mut body = String::new();+ for (name, text) in files {+ body.push_str(&format!(+ "--upload-boundary\r\nContent-Disposition: form-data; name=\"files\"; filename=\"{name}\"\r\nContent-Type: text/plain\r\n\r\n{text}\r\n"+ ));+ }+ body.push_str("--upload-boundary--\r\n");+ let response = self+ .app+ .clone()+ .oneshot(+ Request::post(format!("/api/v1/kbs/{kb}/documents{query}"))+ .header("Authorization", format!("Bearer {}", self.token))+ .header(+ "Content-Type",+ "multipart/form-data; boundary=upload-boundary",+ )+ .body(Body::from(body))?,+ )+ .await?;+ let status = response.status();+ let body = serde_json::from_slice(&to_bytes(response.into_body(), 1024 * 1024).await?)?;+ Ok((status, body))+ }++ async fn created_docs(&self, response: &Value) -> anyhow::Result<Vec<Document>> {+ let mut docs = Vec::new();+ for value in response["created"].as_array().unwrap() {+ let id = serde_json::from_value(value["id"].clone())?;+ let doc = documents::get(&self.pool, id).await?;+ assert_eq!(serde_json::to_value(&doc)?, *value);+ docs.push(doc);+ }+ Ok(docs)+ }++ async fn cleanup(self) -> anyhow::Result<()> {+ sqlx::query("DELETE FROM jobs WHERE payload->>'document_id' IN (SELECT id::text FROM documents WHERE kb_id IN ($1,$2))")+ .bind(self.kb).bind(self.other_kb).execute(&self.pool).await?;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(self.org)+ .execute(&self.pool)+ .await?;+ Ok(())+ }+}++#[tokio::test]+async fn each_upload_keeps_its_own_date_before_processing_and_extraction() -> anyhow::Result<()> {+ let Some(f) = Fixture::new().await? else {+ return Ok(());+ };+ let files = [+ ("one.txt", "2024-02-29\nFirst announcement"),+ ("two.md", "(2025年3月1日)\nSecond announcement"),+ ];+ let (status, response) = f+ .upload(+ f.kb,+ &format!("?source={}", f.folder),+ &[files[0], files[1], files[0], ("empty.txt", "")],+ )+ .await?;+ assert_eq!(status, StatusCode::OK, "{response}");+ let created = f.created_docs(&response).await?;+ assert_eq!(created.len(), 2);+ assert_eq!(response["skipped"].as_array().unwrap().len(), 2);+ let independent = sqlx::PgPool::connect(&utopia_store::test_db::url().unwrap()).await?;+ for (doc, expected) in created+ .iter()+ .zip(["2024-02-29T00:00:00Z", "2025-03-01T00:00:00Z"])+ {+ let date = Some(expected.parse::<DateTime<Utc>>()?);+ assert_eq!(+ (doc.doc_time, doc.doc_time_source.as_str(), doc.source_id),+ (date, "content", Some(f.folder))+ );+ // Claim only this test's job, using an independent connection as a worker would.+ let id: i64 = sqlx::query_scalar(+ "UPDATE jobs SET status='running', locked_at=now() WHERE kind='process_document'+ AND payload->>'document_id'=$1 AND status='queued' AND run_at<=now() RETURNING id",+ )+ .bind(doc.id.to_string())+ .fetch_one(&independent)+ .await?;+ let stored = documents::get(&independent, doc.id).await?;+ assert_eq!(+ (stored.doc_time, stored.doc_time_source.as_str()),+ (date, "content")+ );+ crate::pipeline::process_document(&f.state, doc.id).await?;+ sqlx::query("UPDATE jobs SET status='done' WHERE id=$1")+ .bind(id)+ .execute(&independent)+ .await?;+ let queued: i64 = sqlx::query_scalar(+ "SELECT count(*) FROM jobs WHERE kind='extract_document' AND status='queued' AND payload->>'document_id'=$1"+ ).bind(doc.id.to_string()).fetch_one(&independent).await?;+ assert_eq!(queued, 1);+ let stored = documents::get(&independent, doc.id).await?;+ assert_eq!(+ (stored.doc_time, stored.doc_time_source.as_str()),+ (date, "content")+ );+ }+ let (_, duplicate) = f.upload(f.kb, "", &files).await?;+ assert_eq!(duplicate["created"], json!([]));+ let jobs: i64 = sqlx::query_scalar("SELECT count(*) FROM jobs WHERE kind='process_document' AND payload->>'document_id'=ANY($1)")+ .bind(created.iter().map(|d| d.id.to_string()).collect::<Vec<_>>()).fetch_one(&f.pool).await?;+ assert_eq!(jobs, 2, "duplicates must not queue more processing");+ let (status, other) = f.upload(f.other_kb, "", &[files[0]]).await?;+ assert_eq!(status, StatusCode::OK, "{other}");+ let other = f.created_docs(&other).await?;+ assert_eq!(other.len(), 1);+ assert_ne!(other[0].id, created[0].id);+ assert_eq!(other[0].kb_id, f.other_kb);+ assert_eq!(other[0].doc_time, created[0].doc_time);+ assert_eq!(other[0].doc_time_source, "content");+ f.cleanup().await+}++#[tokio::test]+async fn fallback_restore_and_other_ingest_dates_do_not_change() -> anyhow::Result<()> {+ let Some(f) = Fixture::new().await? else {+ return Ok(());+ };+ let (status, response) = f+ .upload(+ f.kb,+ "",+ &[+ ("no-date.txt", "Announcement without a date"),+ ("invalid.md", "2023-02-29\nImpossible date"),+ ("ambiguous.md", "2024-03-01 / 2025-03-01\nTwo dates"),+ ],+ )+ .await?;+ assert_eq!(status, StatusCode::OK, "{response}");+ let docs = f.created_docs(&response).await?;+ assert_eq!(docs.len(), 3);+ for doc in &docs {+ assert_eq!(doc.doc_time_source, "upload_time");+ assert_eq!(doc.doc_time, Some(doc.created_at));+ }+ // A pre-existing, dated body with upload_time must retain that historical fallback on restoration.+ let body = b"2024-02-29\nHistorical announcement";+ use sha2::{Digest, Sha256};+ let sha = super::hex(&Sha256::digest(body));+ f.state.blob.put(&sha, body).await?;+ let old = documents::create(+ &f.pool,+ f.kb,+ "old.txt",+ "text/plain",+ body.len() as i64,+ &sha,+ None,+ None,+ None,+ )+ .await?;+ documents::delete(&f.pool, f.kb, old.id, None).await?;+ let (status, restored) = f+ .upload(f.kb, "", &[("old.txt", std::str::from_utf8(body)?)])+ .await?;+ assert_eq!(status, StatusCode::OK, "{restored}");+ let restored = f.created_docs(&restored).await?;+ assert_eq!(restored.len(), 1);+ let restored = &restored[0];+ assert_eq!(restored.id, old.id);+ assert_eq!(+ (restored.doc_time, &restored.doc_time_source),+ (old.doc_time, &old.doc_time_source)+ );+ crate::pipeline::process_document(&f.state, old.id).await?;+ let reprocessed = documents::get(&f.pool, old.id).await?;+ assert_eq!(+ (reprocessed.doc_time, reprocessed.doc_time_source),+ (old.doc_time, old.doc_time_source)+ );++ let explicit = "2022-01-01T08:00:00+08:00".parse::<DateTime<Utc>>()?;+ crate::ingest_sources::ingest_upload(+ &f.state,+ f.kb,+ "json.txt",+ "text/plain",+ b"2025-03-01\nJSON",+ Some(explicit),+ )+ .await?;+ crate::ingest_sources::ingest_item(+ &f.state,+ f.kb,+ f.source,+ "api:key",+ "sync.md",+ "text/markdown",+ b"2025-03-01\nSync",+ Some(explicit),+ )+ .await?;+ for name in ["json.txt", "sync.md"] {+ let doc: Document =+ sqlx::query_as("SELECT * FROM documents WHERE kb_id=$1 AND filename=$2")+ .bind(f.kb)+ .bind(name)+ .fetch_one(&f.pool)+ .await?;+ crate::pipeline::process_document(&f.state, doc.id).await?;+ let doc = documents::get(&f.pool, doc.id).await?;+ assert_eq!(+ (doc.doc_time, doc.doc_time_source.as_str()),+ (Some(explicit), "source")+ );+ }+ f.cleanup().await+}++#[tokio::test]+async fn date_detection_keeps_upload_access_and_folder_checks() -> anyhow::Result<()> {+ let Some(f) = Fixture::new().await? else {+ return Ok(());+ };+ for source in [f.foreign_folder, f.source] {+ let (status, body) = f+ .upload(+ f.kb,+ &format!("?source={source}"),+ &[("dated.txt", "2024-03-01")],+ )+ .await?;+ assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY);+ assert_eq!(body["code"], "upload_needs_folder");+ }+ sqlx::query("UPDATE kb_members SET role='viewer' WHERE kb_id=$1 AND user_id=$2")+ .bind(f.kb)+ .bind(f.user)+ .execute(&f.pool)+ .await?;+ let (status, _) = f.upload(f.kb, "", &[("dated.txt", "2024-03-01")]).await?;+ assert_eq!(status, StatusCode::FORBIDDEN);+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM documents WHERE kb_id=$1")+ .bind(f.kb)+ .fetch_one(&f.pool)+ .await?;+ assert_eq!(count, 0);+ f.cleanup().await+}diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rsindex 0bafb544..c45bf405 100644--- a/crates/utopia-store/src/documents.rs+++ b/crates/utopia-store/src/documents.rs@@ -17,6 +17,61 @@ pub async fn create(source_id: Option<Uuid>,doc_time: Option<chrono::DateTime<chrono::Utc>>,external_key: Option<&str>,+) -> AppResult<Document> {+ create_with_time_source(+ pool,+ kb_id,+ filename,+ mime,+ size_bytes,+ sha256,+ source_id,+ doc_time,+ "source",+ external_key,+ )+ .await+}++/// 正文识别只属于文件上传;JSON ingest 和同步传来的日期仍用原来的 source 语义。+#[allow(clippy::too_many_arguments)]+pub async fn create_from_upload(+ pool: &PgPool,+ kb_id: Uuid,+ filename: &str,+ mime: &str,+ size_bytes: i64,+ sha256: &str,+ source_id: Option<Uuid>,+ content_time: Option<DateTime<Utc>>,+) -> AppResult<Document> {+ create_with_time_source(+ pool,+ kb_id,+ filename,+ mime,+ size_bytes,+ sha256,+ source_id,+ content_time,+ "content",+ None,+ )+ .await+}++#[allow(clippy::too_many_arguments)]+async fn create_with_time_source(+ pool: &PgPool,+ kb_id: Uuid,+ filename: &str,+ mime: &str,+ size_bytes: i64,+ sha256: &str,+ source_id: Option<Uuid>,+ doc_time: Option<DateTime<Utc>>,+ time_source: &str,+ external_key: Option<&str>,) -> AppResult<Document> {// 同样的内容回来了,而它只是被删过:复活那一篇,而不是撞 (kb_id, sha256) 的唯一// 索引报「已存在」。撤销删除的一种自然形态——重传就是「我要它回来」(#268)@@ -45,7 +100,7 @@ pub async fn create(.bind(source_id).bind(doc_time).bind(if doc_time.is_some() {- "source"+ time_source} else {"upload_time"})