Skip to content

World's first open-source enterprise world model.

Keep Unicode source filenames from panicking during synchronization (#793)

Maya Kide618d1c8211c
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs
index 10466fdc..dda5a51d 100644
--- a/crates/utopia-server/src/ingest_sources.rs
+++ b/crates/utopia-server/src/ingest_sources.rs
@@ -435,7 +435,7 @@ fn filename_from_url(url: &str, mime: &str) -> String {
.trim_start_matches("https://")
.trim_start_matches("http://")
.trim_end_matches('/');
- let mut slug: String = stripped
+ let slug: String = stripped
.chars()
.map(|c| {
if c.is_alphanumeric() || c == '.' || c == '-' {
@@ -445,7 +445,7 @@ fn filename_from_url(url: &str, mime: &str) -> String {
}
})
.collect();
- slug.truncate(120);
+ let slug = truncate_utf8(&slug, 120);
let has_ext = slug
.rsplit('.')
.next()
@@ -895,11 +895,11 @@ async fn sync_jira_issues(state: &AppState, source: &Source) -> anyhow::Result<S
/// 标题 → 文件名安全的片段。与 RSS 那条路同一个口径(非字母数字换成 -,截断)。
fn slugify(title: &str) -> String {
- let mut s: String = title
+ let s: String = title
.chars()
.map(|c| if c.is_alphanumeric() { c } else { '-' })
.collect();
- s.truncate(60);
+ let s = truncate_utf8(&s, 60);
s.trim_matches('-').to_string()
}
@@ -1208,3 +1208,7 @@ mod tests {
assert!(rss_entry_key(entry, Some("https://example.com/article")).is_some());
}
}
+
+#[cfg(test)]
+#[path = "source_filename_tests.rs"]
+mod source_filename_tests;
diff --git a/crates/utopia-server/src/source_filename_tests.rs b/crates/utopia-server/src/source_filename_tests.rs
new file mode 100644
index 00000000..561fe7be
--- /dev/null
+++ b/crates/utopia-server/src/source_filename_tests.rs
@@ -0,0 +1,98 @@
+//! 文件名截断不能让正常的多字节标题掀掉整次来源同步。
+use super::{filename_from_url, slugify, sync_source};
+use std::sync::Arc;
+use uuid::Uuid;
+use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate};
+
+#[test]
+fn source_filenames_end_on_character_boundaries() {
+ assert_eq!(
+ slugify(&format!("A{}", "中".repeat(30))),
+ format!("A{}", "中".repeat(19))
+ );
+ let name = filename_from_url(
+ &format!("https://example.com/a{}", "中".repeat(50)),
+ "text/html",
+ );
+ assert_eq!(name, format!("example.com-a{}.html", "中".repeat(35)));
+ assert_eq!(slugify(&"a".repeat(80)), "a".repeat(60));
+}
+
+#[tokio::test]
+async fn a_long_unicode_title_does_not_stop_an_rss_sync() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = sqlx::PgPool::connect(&url).await?;
+ let (org, ws, kb, source) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'filename-test')")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'filename-test')")
+ .bind(ws)
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'filename-test')")
+ .bind(kb)
+ .bind(ws)
+ .execute(&pool)
+ .await?;
+ let server = MockServer::start().await;
+ let title = format!("A{}", "中".repeat(30));
+ let feed = format!(
+ r#"<rss version="2.0"><channel><title>Fixture</title><link>https://example.com/</link><description>Fixture</description><item><guid>unicode</guid><title>{title}</title><description>First article</description></item><item><guid>after</guid><title>Next article</title><description>Second article</description></item></channel></rss>"#
+ );
+ Mock::given(method("GET"))
+ .respond_with(ResponseTemplate::new(200).set_body_string(feed))
+ .mount(&server)
+ .await;
+ sqlx::query("INSERT INTO sources(id,kb_id,kind,name,config) VALUES($1,$2,'rss','fixture',$3)")
+ .bind(source)
+ .bind(kb)
+ .bind(serde_json::json!({"feed_url":server.uri()}))
+ .execute(&pool)
+ .await?;
+ let dir = std::env::temp_dir().join(format!("utopia-source-filename-{source}"));
+ let cfg = utopia_core::config::AppConfig {
+ data_dir: dir.to_string_lossy().into_owned(),
+ ..Default::default()
+ };
+ let search = Arc::new(utopia_search::SearchIndex::open(&dir.join("search"))?);
+ let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into());
+ let result = tokio::spawn(async move { sync_source(&state, source).await }).await;
+ let docs: Vec<String> =
+ sqlx::query_scalar("SELECT filename FROM documents WHERE source_id=$1 ORDER BY filename")
+ .bind(source)
+ .fetch_all(&pool)
+ .await?;
+ let status: String = sqlx::query_scalar("SELECT last_sync_status FROM sources WHERE id=$1")
+ .bind(source)
+ .fetch_one(&pool)
+ .await?;
+ sqlx::query("DELETE FROM knowledge_bases WHERE id=$1")
+ .bind(kb)
+ .execute(&pool)
+ .await?;
+ sqlx::query("DELETE FROM organizations WHERE id=$1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ let _ = std::fs::remove_dir_all(dir);
+ result??;
+ assert_eq!(
+ docs,
+ vec![
+ format!("A{}.html", "中".repeat(19)),
+ "Next-article.html".into()
+ ]
+ );
+ assert_eq!(status, "ok");
+ Ok(())
+}