World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
use super::content_time;
use axum::body::{to_bytes, Body};
use axum::http::{HeaderMap, Request, StatusCode};
use chrono::{DateTime, Utc};
use serde_json::{json, Value};
use std::sync::Arc;
use tower::ServiceExt;
use utopia_core::models::Document;
use utopia_store::documents;
use uuid::Uuid;
#[test]
fn only_a_complete_opening_dateline_sets_the_date() {
let expected = "2024-02-29T00:00:00Z".parse::<DateTime<Utc>>().unwrap();
for text in [
"2024-02-29\nAnnouncement",
"\u{feff}\r\n 2024-02-29 \r\nAnnouncement",
"(2024年2月29日)\n公告",
"(2024-02-29)",
] {
for filename in ["filing.txt", "filing.MD", "filing.markdown"] {
assert_eq!(content_time(filename, text.as_bytes()), Some(expected));
}
}
for text in [
"",
" \n\t",
"2023-02-29",
"2024-13-01",
"2024-02",
"02/03/2024",
"24-02-29",
"2024-02-29 / 2025-02-28",
"2024-02-29 was the previous meeting",
"Announcement\n2024-02-29",
"# 2024-02-29 announcement",
"2024-02-29T12:00:00+08:00",
"(2024年2月29日",
] {
assert_eq!(content_time("filing.md", text.as_bytes()), None, "{text:?}");
}
for filename in [
"filing.pdf",
"filing.docx",
"filing.csv",
"2024-02-29",
"filing",
] {
assert_eq!(content_time(filename, b"2024-02-29"), None);
}
}
/// #610:文件名里的日期和正文一样,但要排在前面——公告语料通常按日期命名,
/// multipart 解析时这条信息已经在那儿了;正文里写「2024年2月29日 公告」还是
/// 文件名叫「2024-02-29_gaoshu.txt」,对读的人来说是一个东西。
///
/// 日期格式与正文一致:ISO `YYYY-MM-DD` 或中文 `YYYY年M月D日`(月/日 1-2 位都行,
/// 输出一律是规范化的两位数)。文件名后跟任意内容(包括完全没有后跟),
/// 只要日期在主名开头就算。
#[test]
fn a_filename_with_a_leading_date_sets_the_date_even_when_body_has_none() {
let expected = "2024-02-29T00:00:00Z".parse::<DateTime<Utc>>().unwrap();
// ISO
for filename in [
"2024-02-29.txt",
"2024-02-29_filing.txt",
"2024-02-29.md",
"2024-02-29.markdown",
"2024-02-29 some filing.txt", // 后跟空格分隔
] {
assert_eq!(
content_time(filename, b""),
Some(expected),
"filename {filename:?}"
);
}
// ISO 月/日 1 位 → 标准化为 2 位
let expected_0209 = "2024-02-09T00:00:00Z".parse::<DateTime<Utc>>().unwrap();
assert_eq!(content_time("2024-2-9 filing.md", b""), Some(expected_0209));
// 中文
for filename in ["2024年2月29日.txt", "2024年02月29日_公告.md"] {
assert_eq!(
content_time(filename, b""),
Some(expected),
"filename {filename:?}"
);
}
// 中文月/日 1 位 → 标准化为 2 位
let expected_0209 = "2024-02-09T00:00:00Z".parse::<DateTime<Utc>>().unwrap();
assert_eq!(
content_time("2024年2月9日 公告.markdown", b""),
Some(expected_0209)
);
}
/// 文件名没日期时,回到正文找:与正文解析路径完全一致(`#610` 的「文件名优先,
/// 正文兜底」)
#[test]
fn a_filename_without_a_date_falls_back_to_body() {
let expected = "2024-02-29T00:00:00Z".parse::<DateTime<Utc>>().unwrap();
// ISO in body
assert_eq!(
content_time("filing.txt", b"2024-02-29\nAnnouncement"),
Some(expected)
);
// 中文 in body
assert_eq!(
content_time("filing.md", "(2024年2月29日)\n公告".as_bytes()),
Some(expected)
);
// Filename 无日期,正文也无日期 → 全部 None
assert_eq!(content_time("filing.txt", b""), None);
assert_eq!(content_time("filing.txt", b"Announcement"), None);
}
/// 模糊/部分/无日期:全走 None(与正文那条路径同一套规矩)
#[test]
fn a_partial_or_ambiguous_filename_does_not_match() {
// 不是 4 位数年份开头
for filename in [
"24-02-29.txt", // 年份只有 2 位
"abcd-02-29.txt", // 年份是字母
" 2024-02-29.txt", // 开头有空格
] {
assert_eq!(content_time(filename, b""), None, "filename {filename:?}");
}
// 日期格式对但数字超出范围(chrono 拒)
for filename in [
"2023-02-29.txt", // 2023 不是闰年
"2024-13-01.txt", // 月份 13
"2024-02-30.txt", // 2 月 30 日
"2024年13月01日.txt",
] {
assert_eq!(content_time(filename, b""), None, "filename {filename:?}");
}
// 中文格式部分缺
for filename in [
"2024年.txt", // 缺月日
"2024年2月.txt", // 缺日
"2024年2月29.txt", // 缺日字
] {
assert_eq!(content_time(filename, b""), None, "filename {filename:?}");
}
}
/// 文件名 vs 正文优先级:文件名日期早于正文日期时,**用文件名的**
/// (announcement corpus 多半以日期命名)。
#[test]
fn filename_date_wins_over_body_date() {
let filename_date = "2024-02-29T00:00:00Z".parse::<DateTime<Utc>>().unwrap();
// 正文是另一个日期;文件名日期优先
assert_eq!(
content_time("2024-02-29_filing.txt", b"2025-12-31\nOld announcement"),
Some(filename_date)
);
}
impl Fixture {
async fn get_raw(
&self,
path: &str,
token: Option<&str>,
) -> anyhow::Result<(StatusCode, HeaderMap, Vec<u8>)> {
let mut request = Request::get(path);
if let Some(token) = token {
request = request.header("Authorization", format!("Bearer {token}"));
}
let response = self
.app
.clone()
.oneshot(request.body(Body::empty())?)
.await?;
let status = response.status();
let headers = response.headers().clone();
let bytes = to_bytes(response.into_body(), 128 * 1024 * 1024)
.await?
.to_vec();
Ok((status, headers, bytes))
}
async fn get_json(
&self,
path: &str,
token: Option<&str>,
) -> anyhow::Result<(StatusCode, Value)> {
let (status, _, bytes) = self.get_raw(path, token).await?;
Ok((status, serde_json::from_slice(&bytes)?))
}
}
fn content_headers(headers: &HeaderMap, sha256: &str, size: usize) {
assert_eq!(
headers["content-type"], "application/x-audit-record",
"the ledger's MIME, not a guessed type"
);
assert_eq!(headers["content-length"], size.to_string());
assert_eq!(headers["etag"], format!("\"{sha256}\""));
assert_eq!(
headers["content-disposition"],
"attachment; filename=\"audit.bin\"; filename*=UTF-8''audit.bin"
);
}
#[tokio::test]
async fn document_content_serves_the_current_and_recorded_versions() -> anyhow::Result<()> {
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let original = b"generation one".to_vec();
let doc = f.create_retained(f.kb, &original).await?;
let path = format!("/api/v1/documents/{}/content", doc.id);
let (status, headers, bytes) = f.get_raw(&path, Some(&f.token)).await?;
assert_eq!(
status,
StatusCode::OK,
"{}",
String::from_utf8_lossy(&bytes)
);
assert_eq!(bytes, original);
content_headers(&headers, &doc.sha256, original.len());
let revised = b"generation two has grown".to_vec();
use sha2::{Digest, Sha256};
let revised_sha = super::hex(&Sha256::digest(&revised));
f.state.blob.put(&revised_sha, &revised).await?;
documents::replace_content_and_enqueue_processing(
&f.pool,
doc.id,
&doc.filename,
&doc.mime,
revised.len() as i64,
&revised_sha,
None,
)
.await?;
let (status, headers, bytes) = f
.get_raw(&format!("{path}?version=2"), Some(&f.token))
.await?;
assert_eq!(
status,
StatusCode::OK,
"{}",
String::from_utf8_lossy(&bytes)
);
assert_eq!(bytes, revised);
content_headers(&headers, &revised_sha, revised.len());
let (status, headers, bytes) = f
.get_raw(&format!("{path}?version=1"), Some(&f.token))
.await?;
assert_eq!(
status,
StatusCode::OK,
"{}",
String::from_utf8_lossy(&bytes)
);
assert_eq!(bytes, original);
content_headers(&headers, &doc.sha256, original.len());
f.cleanup().await
}
#[tokio::test]
async fn versions_ledger_names_what_content_can_serve() -> anyhow::Result<()> {
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let original = b"auditable history";
let doc = f.create_retained(f.kb, original).await?;
let (status, body) = f
.get_json(
&format!("/api/v1/documents/{}/versions", doc.id),
Some(&f.token),
)
.await?;
assert_eq!(status, StatusCode::OK, "{body}");
let versions = body["versions"].as_array().expect("version ledger");
assert_eq!(versions.len(), 1);
assert_eq!(versions[0]["version"], 1);
assert_eq!(versions[0]["sha256"], doc.sha256);
assert_eq!(versions[0]["size_bytes"], original.len() as i64);
assert!(versions[0]["ingested_at"].is_string());
let path = format!("/api/v1/documents/{}/content?version=99", doc.id);
let (status, _, bytes) = f.get_raw(&path, Some(&f.token)).await?;
assert_eq!(status, StatusCode::NOT_FOUND, "{:?}", bytes);
f.cleanup().await
}
#[tokio::test]
async fn deleted_bytes_stay_readable_and_purged_tombstones_answer_gone() -> anyhow::Result<()> {
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let original = b"retained after deletion";
let doc = f.create_retained(f.kb, original).await?;
documents::delete(&f.pool, f.kb, doc.id, None).await?;
let path = format!("/api/v1/documents/{}/content", doc.id);
let (status, _, bytes) = f.get_raw(&path, Some(&f.token)).await?;
assert_eq!(
status,
StatusCode::OK,
"{}",
String::from_utf8_lossy(&bytes)
);
assert_eq!(bytes, original);
documents::purge(&f.pool, f.kb, doc.id).await?;
let (status, _, bytes) = f.get_raw(&path, Some(&f.token)).await?;
assert_eq!(status, StatusCode::GONE, "{:?}", bytes);
let versions_path = format!("/api/v1/documents/{}/versions", doc.id);
let (status, _, bytes) = f.get_raw(&versions_path, Some(&f.token)).await?;
assert_eq!(status, StatusCode::GONE, "{:?}", bytes);
f.cleanup().await
}
#[tokio::test]
async fn a_ledger_referenced_missing_blob_is_an_invariant_failure() -> anyhow::Result<()> {
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let (status, created) = f
.upload(f.kb, "", &[("audit.bin", "still promised")])
.await?;
assert_eq!(status, StatusCode::OK, "{created}");
let doc = f.created_docs(&created).await?.remove(0);
f.state.blob.delete(&doc.sha256).await?;
let path = format!("/api/v1/documents/{}/content", doc.id);
let (status, _, bytes) = f.get_raw(&path, Some(&f.token)).await?;
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{:?}", bytes);
f.cleanup().await
}
#[tokio::test]
async fn content_reads_keep_viewer_access_pat_scope_and_reject_source_tokens() -> anyhow::Result<()>
{
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let doc = f.create_retained(f.kb, b"scoped bytes").await?;
let content_path = format!("/api/v1/documents/{}/content", doc.id);
let versions_path = format!("/api/v1/documents/{}/versions", doc.id);
sqlx::query("UPDATE kb_members SET role='viewer' WHERE kb_id=$1 AND user_id=$2")
.bind(f.kb)
.bind(f.user)
.execute(&f.pool)
.await?;
let (status, _, bytes) = f.get_raw(&content_path, Some(&f.token)).await?;
assert_eq!(
status,
StatusCode::OK,
"{}",
String::from_utf8_lossy(&bytes)
);
let (_, pat) =
utopia_store::tokens::issue(&f.pool, f.user, "audit", "read", None, None).await?;
for path in [&content_path, &versions_path] {
let (status, _, bytes) = f.get_raw(path, Some(&pat)).await?;
assert_eq!(
status,
StatusCode::OK,
"{}",
String::from_utf8_lossy(&bytes)
);
}
let (_, scoped_pat) = utopia_store::tokens::issue(
&f.pool,
f.user,
"other base only",
"read",
Some(&[f.other_kb]),
None,
)
.await?;
let (status, _, bytes) = f.get_raw(&content_path, Some(&scoped_pat)).await?;
assert_eq!(status, StatusCode::NOT_FOUND, "{:?}", bytes);
let source_token = crate::api::sources_routes::new_ingest_token();
let (status, _, bytes) = f.get_raw(&content_path, Some(&source_token)).await?;
assert_eq!(status, StatusCode::UNAUTHORIZED, "{:?}", bytes);
f.cleanup().await
}
#[test]
fn header_decoding_is_bounded_and_never_accepts_a_truncated_line() {
let expected = "2024-02-29T00:00:00Z".parse::<DateTime<Utc>>().unwrap();
let mut utf16 = vec![0xff, 0xfe];
for unit in "(2024年2月29日)\r\n公告".encode_utf16() {
utf16.extend(unit.to_le_bytes());
}
assert_eq!(content_time("filing.txt", &utf16), Some(expected));
let mut long = b"2024-02-29\n".to_vec();
long.extend("正文".repeat(2000).as_bytes());
assert_eq!(content_time("filing.md", &long), Some(expected));
let truncated = format!(
"{}2024-02-29 is an event, not a dateline",
"\n".repeat(4086)
);
assert_eq!(content_time("filing.txt", truncated.as_bytes()), None);
let too_late = format!("{}2024-02-29\n", "\n".repeat(4096));
assert_eq!(content_time("filing.txt", too_late.as_bytes()), None);
}
struct Fixture {
pool: sqlx::PgPool,
state: crate::state::AppState,
app: axum::Router,
org: Uuid,
kb: Uuid,
other_kb: Uuid,
user: Uuid,
folder: Uuid,
source: Uuid,
token: String,
_dir: tempfile::TempDir,
}
impl Fixture {
async fn new() -> anyhow::Result<Option<Self>> {
let Some(url) = utopia_store::test_db::url() else {
return Ok(None);
};
let pool = sqlx::PgPool::connect(&url).await?;
utopia_store::db::migrate(&pool).await?;
let (org, ws, kb, other_kb, user, folder, foreign_folder, source) = (
Uuid::now_v7(),
Uuid::now_v7(),
Uuid::now_v7(),
Uuid::now_v7(),
Uuid::now_v7(),
Uuid::now_v7(),
Uuid::now_v7(),
Uuid::now_v7(),
);
// Only locally generated UUIDs are interpolated into fixture SQL.
sqlx::raw_sql(&format!(
"INSERT INTO organizations(id,name) VALUES ('{org}','upload-date-test');
INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','upload-date-test');
INSERT INTO users(id,org_id,email,display_name,password_hash)
VALUES ('{user}','{org}','{user}@upload.test','upload-test','unused');
INSERT INTO knowledge_bases(id,workspace_id,name) VALUES
('{kb}','{ws}','uploads'), ('{other_kb}','{ws}','other-uploads');
INSERT INTO kb_members(kb_id,user_id,role) VALUES
('{kb}','{user}','editor'), ('{other_kb}','{user}','editor');
INSERT INTO sources(id,kb_id,kind,name) VALUES
('{folder}','{kb}','folder','uploads'),
('{foreign_folder}','{other_kb}','folder','other-folder'),
('{source}','{kb}','api','api-source');"
))
.execute(&pool)
.await?;
// Processing may queue extraction, but no test calls a live model.
utopia_store::settings::upsert(
&pool,
ws,
Some("http://127.0.0.1:9"),
None,
Some("test-model"),
None,
None,
None,
None,
)
.await?;
let dir = tempfile::tempdir()?;
let cfg = utopia_core::config::AppConfig {
data_dir: dir.path().to_string_lossy().into_owned(),
..Default::default()
};
let search = Arc::new(utopia_search::SearchIndex::open(
&dir.path().join("search"),
)?);
let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into());
let token = crate::auth::issue_token(&state, user)?;
let app = super::super::router(state.clone(), &cfg);
Ok(Some(Self {
pool,
state,
app,
org,
kb,
other_kb,
user,
folder,
source,
token,
_dir: dir,
}))
}
async fn create_retained(&self, kb: Uuid, bytes: &[u8]) -> anyhow::Result<Document> {
use sha2::{Digest, Sha256};
let sha256 = super::hex(&Sha256::digest(bytes));
self.state.blob.put(&sha256, bytes).await?;
Ok(documents::create_with_version_and_processing(
&self.pool,
kb,
"audit.bin",
"application/x-audit-record",
bytes.len() as i64,
&sha256,
None,
None,
None,
)
.await?)
}
async fn upload(
&self,
kb: Uuid,
query: &str,
files: &[(&str, &str)],
) -> anyhow::Result<(StatusCode, Value)> {
let mut body = String::new();
for (name, text) in files {
body.push_str(&format!(
"--upload-boundary\r\nContent-Disposition: form-data; name=\"files\"; filename=\"{name}\"\r\nContent-Type: text/plain\r\n\r\n{text}\r\n"
));
}
body.push_str("--upload-boundary--\r\n");
let response = self
.app
.clone()
.oneshot(
Request::post(format!("/api/v1/kbs/{kb}/documents{query}"))
.header("Authorization", format!("Bearer {}", self.token))
.header(
"Content-Type",
"multipart/form-data; boundary=upload-boundary",
)
.body(Body::from(body))?,
)
.await?;
let status = response.status();
let body = serde_json::from_slice(&to_bytes(response.into_body(), 1024 * 1024).await?)?;
Ok((status, body))
}
async fn created_docs(&self, response: &Value) -> anyhow::Result<Vec<Document>> {
let mut docs = Vec::new();
for value in response["created"].as_array().unwrap() {
let id = serde_json::from_value(value["id"].clone())?;
let doc = documents::get(&self.pool, id).await?;
assert_eq!(serde_json::to_value(&doc)?, *value);
docs.push(doc);
}
Ok(docs)
}
async fn cleanup(self) -> anyhow::Result<()> {
sqlx::query("DELETE FROM jobs WHERE payload->>'document_id' IN (SELECT id::text FROM documents WHERE kb_id IN ($1,$2))")
.bind(self.kb).bind(self.other_kb).execute(&self.pool).await?;
sqlx::query("DELETE FROM organizations WHERE id=$1")
.bind(self.org)
.execute(&self.pool)
.await?;
Ok(())
}
}
#[tokio::test]
async fn each_upload_keeps_its_own_date_before_processing_and_extraction() -> anyhow::Result<()> {
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let files = [
("one.txt", "2024-02-29\nFirst announcement"),
("two.md", "(2025年3月1日)\nSecond announcement"),
];
let (status, response) = f
.upload(
f.kb,
&format!("?source={}", f.folder),
&[files[0], files[1], files[0], ("empty.txt", "")],
)
.await?;
assert_eq!(status, StatusCode::OK, "{response}");
let created = f.created_docs(&response).await?;
assert_eq!(created.len(), 2);
assert_eq!(response["skipped"].as_array().unwrap().len(), 2);
let independent = sqlx::PgPool::connect(&utopia_store::test_db::url().unwrap()).await?;
for (doc, expected) in created
.iter()
.zip(["2024-02-29T00:00:00Z", "2025-03-01T00:00:00Z"])
{
let date = Some(expected.parse::<DateTime<Utc>>()?);
assert_eq!(
(doc.doc_time, doc.doc_time_source.as_str(), doc.source_id),
(date, "content", Some(f.folder))
);
let versions = documents::versions(&independent, doc.id).await?;
assert_eq!(versions.len(), 1);
assert_eq!(versions[0].version, 1);
assert_eq!(versions[0].sha256, doc.sha256);
assert_eq!(versions[0].size_bytes, doc.size_bytes);
let version_date: Option<DateTime<Utc>> = sqlx::query_scalar(
"SELECT doc_time FROM document_versions WHERE document_id=$1 AND version=1",
)
.bind(doc.id)
.fetch_one(&independent)
.await?;
assert_eq!(version_date, date);
let original = files
.iter()
.find(|(name, _)| *name == doc.filename)
.unwrap()
.1;
for suffix in ["", "?version=1"] {
let (status, headers, bytes) = f
.get_raw(
&format!("/api/v1/documents/{}/content{suffix}", doc.id),
Some(&f.token),
)
.await?;
assert_eq!(status, StatusCode::OK);
assert_eq!(headers["content-type"], "text/plain");
assert_eq!(bytes, original.as_bytes());
}
// Claim only this test's job, using an independent connection as a worker would.
let id: i64 = sqlx::query_scalar(
"UPDATE jobs SET status='running', locked_at=now() WHERE kind='process_document'
AND payload->>'document_id'=$1 AND status='queued' AND run_at<=now() RETURNING id",
)
.bind(doc.id.to_string())
.fetch_one(&independent)
.await?;
let stored = documents::get(&independent, doc.id).await?;
assert_eq!(
(stored.doc_time, stored.doc_time_source.as_str()),
(date, "content")
);
crate::pipeline::process_document(&f.state, doc.id).await?;
sqlx::query("UPDATE jobs SET status='done' WHERE id=$1")
.bind(id)
.execute(&independent)
.await?;
let queued: i64 = sqlx::query_scalar(
"SELECT count(*) FROM jobs WHERE kind='extract_document' AND status='queued' AND payload->>'document_id'=$1"
).bind(doc.id.to_string()).fetch_one(&independent).await?;
assert_eq!(queued, 1);
let stored = documents::get(&independent, doc.id).await?;
assert_eq!(
(stored.doc_time, stored.doc_time_source.as_str()),
(date, "content")
);
}
let (_, duplicate) = f.upload(f.kb, "", &files).await?;
assert_eq!(duplicate["created"], json!([]));
let jobs: i64 = sqlx::query_scalar("SELECT count(*) FROM jobs WHERE kind='process_document' AND payload->>'document_id'=ANY($1)")
.bind(created.iter().map(|d| d.id.to_string()).collect::<Vec<_>>()).fetch_one(&f.pool).await?;
assert_eq!(jobs, 2, "duplicates must not queue more processing");
let (status, other) = f.upload(f.other_kb, "", &[files[0]]).await?;
assert_eq!(status, StatusCode::OK, "{other}");
let other = f.created_docs(&other).await?;
assert_eq!(other.len(), 1);
assert_ne!(other[0].id, created[0].id);
assert_eq!(other[0].kb_id, f.other_kb);
assert_eq!(other[0].doc_time, created[0].doc_time);
assert_eq!(other[0].doc_time_source, "content");
f.cleanup().await
}
#[tokio::test]
async fn reupload_restores_one_version_and_requeues_processing() -> anyhow::Result<()> {
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let files = [("undated.txt", "An original without a date")];
let (_, body) = f.upload(f.kb, "", &files).await?;
let original = f.created_docs(&body).await?.remove(0);
assert_eq!(original.doc_time_source, "none");
documents::delete(&f.pool, f.kb, original.id, None).await?;
let (status, body) = f.upload(f.kb, "", &files).await?;
assert_eq!(status, StatusCode::OK);
let restored = f.created_docs(&body).await?.remove(0);
assert_eq!(restored.id, original.id);
assert_eq!(restored.doc_time_source, "none");
assert_eq!(documents::versions(&f.pool, restored.id).await?.len(), 1);
let jobs: i64 = sqlx::query_scalar(
"SELECT count(*) FROM jobs WHERE kind='process_document' AND payload->>'document_id'=$1",
)
.bind(restored.id.to_string())
.fetch_one(&f.pool)
.await?;
assert_eq!(jobs, 2);
let (status, _, bytes) = f
.get_raw(
&format!("/api/v1/documents/{}/content", restored.id),
Some(&f.token),
)
.await?;
assert_eq!(status, StatusCode::OK);
assert_eq!(bytes, files[0].1.as_bytes());
f.cleanup().await
}
#[tokio::test]
async fn a_pushed_document_serves_its_original() -> anyhow::Result<()> {
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let pushed = b"A document pushed as JSON";
crate::ingest_sources::ingest_upload(
&f.state,
f.kb,
"pushed.md",
"text/markdown",
pushed,
None,
)
.await?;
let id: Uuid =
sqlx::query_scalar("SELECT id FROM documents WHERE kb_id=$1 AND filename='pushed.md'")
.bind(f.kb)
.fetch_one(&f.pool)
.await?;
assert_eq!(documents::versions(&f.pool, id).await?.len(), 1);
let (status, _, bytes) = f
.get_raw(&format!("/api/v1/documents/{id}/content"), Some(&f.token))
.await?;
assert_eq!(status, StatusCode::OK);
assert_eq!(bytes, pushed);
f.cleanup().await
}
#[tokio::test]
async fn failed_upload_queue_rolls_back_creation_and_restoration() -> anyhow::Result<()> {
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let original = [("original.txt", "Retained original")];
let (_, body) = f.upload(f.kb, "", &original).await?;
let doc = f.created_docs(&body).await?.remove(0);
documents::delete(&f.pool, f.kb, doc.id, None).await?;
let guard = format!("fail_upload_queue_{}", f.kb.simple());
// Reject only this fixture's jobs, after the document and version were written.
sqlx::raw_sql(&format!(
"CREATE FUNCTION {guard}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
IF NEW.kind='process_document' AND EXISTS
(SELECT 1 FROM documents WHERE id::text=NEW.payload->>'document_id' AND kb_id='{}')
THEN RAISE EXCEPTION 'fixture rejects upload queue'; END IF;
RETURN NEW; END $$;
CREATE TRIGGER {guard} BEFORE INSERT ON jobs FOR EACH ROW EXECUTE FUNCTION {guard}();",
f.kb
))
.execute(&f.pool)
.await?;
let result = async {
for files in [[("new.txt", "A new original")], original] {
let (status, _) = f.upload(f.kb, "", &files).await?;
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
}
let state: (i64, i64, i64) = sqlx::query_as(
"SELECT (SELECT count(*) FROM documents WHERE kb_id=$1),
(SELECT count(*) FROM document_versions v JOIN documents d ON d.id=v.document_id WHERE d.kb_id=$1),
(SELECT count(*) FROM jobs WHERE payload->>'document_id'=$2)",
)
.bind(f.kb)
.bind(doc.id.to_string())
.fetch_one(&f.pool)
.await?;
assert_eq!(state, (1, 1, 1));
assert!(documents::get(&f.pool, doc.id).await?.deleted_at.is_some());
Ok::<(), anyhow::Error>(())
}
.await;
sqlx::raw_sql(&format!(
"DROP TRIGGER {guard} ON jobs; DROP FUNCTION {guard}();"
))
.execute(&f.pool)
.await?;
f.cleanup().await?;
result
}
/// `#610`:文件名里有日期时,日期来自文件名而不是正文——这一档要把
/// `content_time` 走文件名优先的路径整条接到数据库层(不只是单测)。
///
/// 这里只挂纯文本的三种情形:ISO 文件名、中文文件名、ISO 文件名 + 中文正文。
/// 它们都对得上 `content_time` 单测,但落库之后 `doc_time` 与 `doc_time_source`
/// 必须一起被设成对应值,不能再回到「正文解析失败 → 全 None」。
#[tokio::test]
async fn a_filename_with_a_date_drives_doc_time_when_body_has_none() -> anyhow::Result<()> {
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let files = [
// ISO 文件名 + 正文无日期
("2024-02-29_filing.txt", "First announcement"),
// 中文文件名 + 正文无日期
("2025年3月1日_公告.md", "第二次公告"),
// ISO 文件名 + 中文正文
(
"2026-06-15_meeting.markdown",
"(2027年1月1日)\nthis header date must not win over the filename",
),
];
let (status, response) = f.upload(f.kb, "", &files).await?;
assert_eq!(status, StatusCode::OK, "{response}");
let created = f.created_docs(&response).await?;
assert_eq!(created.len(), 3);
let expected = [
"2024-02-29T00:00:00Z",
"2025-03-01T00:00:00Z",
"2026-06-15T00:00:00Z",
];
for (doc, expected) in created.iter().zip(expected) {
assert_eq!(
(doc.doc_time, doc.doc_time_source.as_str()),
(Some(expected.parse::<DateTime<Utc>>()?), "content"),
"filename doc {}",
doc.filename,
);
}
f.cleanup().await
}
#[tokio::test]
async fn fallback_restore_and_other_ingest_dates_do_not_change() -> anyhow::Result<()> {
let Some(f) = Fixture::new().await? else {
return Ok(());
};
let (status, response) = f
.upload(
f.kb,
"",
&[
("no-date.txt", "Announcement without a date"),
("invalid.md", "2023-02-29\nImpossible date"),
("ambiguous.md", "2024-03-01 / 2025-03-01\nTwo dates"),
],
)
.await?;
assert_eq!(status, StatusCode::OK, "{response}");
let docs = f.created_docs(&response).await?;
assert_eq!(docs.len(), 3);
// 上传时刻不是文档的日期(0045 决定 3,#714):没有日期就是没有日期
for doc in &docs {
assert_eq!(doc.doc_time_source, "none");
assert_eq!(doc.doc_time, None);
}
// A pre-existing, dated body with upload_time must retain that historical fallback on restoration.
let body = b"2024-02-29\nHistorical announcement";
use sha2::{Digest, Sha256};
let sha = super::hex(&Sha256::digest(body));
f.state.blob.put(&sha, body).await?;
let old = documents::create(
&f.pool,
f.kb,
"old.txt",
"text/plain",
body.len() as i64,
&sha,
None,
None,
None,
)
.await?;
documents::delete(&f.pool, f.kb, old.id, None).await?;
let (status, restored) = f
.upload(f.kb, "", &[("old.txt", std::str::from_utf8(body)?)])
.await?;
assert_eq!(status, StatusCode::OK, "{restored}");
let restored = f.created_docs(&restored).await?;
assert_eq!(restored.len(), 1);
let restored = &restored[0];
assert_eq!(restored.id, old.id);
assert_eq!(
(restored.doc_time, &restored.doc_time_source),
(old.doc_time, &old.doc_time_source)
);
crate::pipeline::process_document(&f.state, old.id).await?;
let reprocessed = documents::get(&f.pool, old.id).await?;
assert_eq!(
(reprocessed.doc_time, reprocessed.doc_time_source),
(old.doc_time, old.doc_time_source)
);
let explicit = "2022-01-01T08:00:00+08:00".parse::<DateTime<Utc>>()?;
crate::ingest_sources::ingest_upload(
&f.state,
f.kb,
"json.txt",
"text/plain",
b"2025-03-01\nJSON",
Some(explicit),
)
.await?;
crate::ingest_sources::ingest_item(
&f.state,
f.kb,
f.source,
"api:key",
"sync.md",
"text/markdown",
b"2025-03-01\nSync",
Some(explicit),
)
.await?;
for name in ["json.txt", "sync.md"] {
let doc: Document =
sqlx::query_as("SELECT * FROM documents WHERE kb_id=$1 AND filename=$2")
.bind(f.kb)
.bind(name)
.fetch_one(&f.pool)
.await?;
crate::pipeline::process_document(&f.state, doc.id).await?;
let doc = documents::get(&f.pool, doc.id).await?;
assert_eq!(
(doc.doc_time, doc.doc_time_source.as_str()),
(Some(explicit), "source")
);
}
f.cleanup().await
}