World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Keep Unicode source filenames from panicking during synchronization (#793)
Maya Kide618d1c8211c
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rsindex 10466fdc..dda5a51d 100644--- a/crates/utopia-server/src/ingest_sources.rs+++ b/crates/utopia-server/src/ingest_sources.rs@@ -435,7 +435,7 @@ fn filename_from_url(url: &str, mime: &str) -> String {.trim_start_matches("https://").trim_start_matches("http://").trim_end_matches('/');- let mut slug: String = stripped+ let slug: String = stripped.chars().map(|c| {if c.is_alphanumeric() || c == '.' || c == '-' {@@ -445,7 +445,7 @@ fn filename_from_url(url: &str, mime: &str) -> String {}}).collect();- slug.truncate(120);+ let slug = truncate_utf8(&slug, 120);let has_ext = slug.rsplit('.').next()@@ -895,11 +895,11 @@ async fn sync_jira_issues(state: &AppState, source: &Source) -> anyhow::Result<S/// 标题 → 文件名安全的片段。与 RSS 那条路同一个口径(非字母数字换成 -,截断)。fn slugify(title: &str) -> String {- let mut s: String = title+ let s: String = title.chars().map(|c| if c.is_alphanumeric() { c } else { '-' }).collect();- s.truncate(60);+ let s = truncate_utf8(&s, 60);s.trim_matches('-').to_string()}@@ -1208,3 +1208,7 @@ mod tests {assert!(rss_entry_key(entry, Some("https://example.com/article")).is_some());}}++#[cfg(test)]+#[path = "source_filename_tests.rs"]+mod source_filename_tests;diff --git a/crates/utopia-server/src/source_filename_tests.rs b/crates/utopia-server/src/source_filename_tests.rsnew file mode 100644index 00000000..561fe7be--- /dev/null+++ b/crates/utopia-server/src/source_filename_tests.rs@@ -0,0 +1,98 @@+//! 文件名截断不能让正常的多字节标题掀掉整次来源同步。+use super::{filename_from_url, slugify, sync_source};+use std::sync::Arc;+use uuid::Uuid;+use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate};++#[test]+fn source_filenames_end_on_character_boundaries() {+ assert_eq!(+ slugify(&format!("A{}", "中".repeat(30))),+ format!("A{}", "中".repeat(19))+ );+ let name = filename_from_url(+ &format!("https://example.com/a{}", "中".repeat(50)),+ "text/html",+ );+ assert_eq!(name, format!("example.com-a{}.html", "中".repeat(35)));+ assert_eq!(slugify(&"a".repeat(80)), "a".repeat(60));+}++#[tokio::test]+async fn a_long_unicode_title_does_not_stop_an_rss_sync() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = sqlx::PgPool::connect(&url).await?;+ let (org, ws, kb, source) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'filename-test')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'filename-test')")+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'filename-test')")+ .bind(kb)+ .bind(ws)+ .execute(&pool)+ .await?;+ let server = MockServer::start().await;+ let title = format!("A{}", "中".repeat(30));+ let feed = format!(+ r#"<rss version="2.0"><channel><title>Fixture</title><link>https://example.com/</link><description>Fixture</description><item><guid>unicode</guid><title>{title}</title><description>First article</description></item><item><guid>after</guid><title>Next article</title><description>Second article</description></item></channel></rss>"#+ );+ Mock::given(method("GET"))+ .respond_with(ResponseTemplate::new(200).set_body_string(feed))+ .mount(&server)+ .await;+ sqlx::query("INSERT INTO sources(id,kb_id,kind,name,config) VALUES($1,$2,'rss','fixture',$3)")+ .bind(source)+ .bind(kb)+ .bind(serde_json::json!({"feed_url":server.uri()}))+ .execute(&pool)+ .await?;+ let dir = std::env::temp_dir().join(format!("utopia-source-filename-{source}"));+ let cfg = utopia_core::config::AppConfig {+ data_dir: dir.to_string_lossy().into_owned(),+ ..Default::default()+ };+ let search = Arc::new(utopia_search::SearchIndex::open(&dir.join("search"))?);+ let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into());+ let result = tokio::spawn(async move { sync_source(&state, source).await }).await;+ let docs: Vec<String> =+ sqlx::query_scalar("SELECT filename FROM documents WHERE source_id=$1 ORDER BY filename")+ .bind(source)+ .fetch_all(&pool)+ .await?;+ let status: String = sqlx::query_scalar("SELECT last_sync_status FROM sources WHERE id=$1")+ .bind(source)+ .fetch_one(&pool)+ .await?;+ sqlx::query("DELETE FROM knowledge_bases WHERE id=$1")+ .bind(kb)+ .execute(&pool)+ .await?;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ let _ = std::fs::remove_dir_all(dir);+ result??;+ assert_eq!(+ docs,+ vec![+ format!("A{}.html", "中".repeat(19)),+ "Next-article.html".into()+ ]+ );+ assert_eq!(status, "ok");+ Ok(())+}