World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Keep failed sync attempts from skipping incremental source updates (#794)
Maya Kid84920bdf38c4
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rsindex dda5a51d..ab34d192 100644--- a/crates/utopia-server/src/ingest_sources.rs+++ b/crates/utopia-server/src/ingest_sources.rs@@ -11,6 +11,10 @@ use utopia_core::models::Source;use utopia_core::models::SourceKind;use uuid::Uuid;+#[cfg(test)]+#[path = "source_checkpoint_tests.rs"]+mod source_checkpoint_tests;+/// 单次同步的新文档上限(防超长 feed/URL 列表拖垮任务)const MAX_NEW_PER_SYNC: usize = 200;const MAX_FEED_BYTES: usize = 4 * 1024 * 1024;@@ -62,17 +66,24 @@ impl SyncStats {pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<()> {let source = utopia_store::sources::get(&state.pool, source_id).await?;+ let kind = SourceKind::parse(&source.kind);+ let since = match kind {+ Some(SourceKind::Custom | SourceKind::GithubIssues | SourceKind::JiraIssues) => {+ utopia_store::sources::last_successful_sync_start(&state.pool, source_id).await?+ }+ _ => None,+ };utopia_store::sources::mark_running(&state.pool, source_id).await?;let run_id = utopia_store::sources::start_run(&state.pool, source_id).await?;state.emit_source(source.kb_id);// 按枚举穷举:加一种来源就得在这里决定它怎么同步,编译器不放过漏掉的那一支- let outcome = match SourceKind::parse(&source.kind) {+ let outcome = match kind {Some(SourceKind::Url) => sync_urls(state, &source).await,Some(SourceKind::Rss) => sync_rss(state, &source).await,- Some(SourceKind::Custom) => sync_custom(state, &source).await,- Some(SourceKind::GithubIssues) => sync_github_issues(state, &source).await,- Some(SourceKind::JiraIssues) => sync_jira_issues(state, &source).await,+ Some(SourceKind::Custom) => sync_custom(state, &source, since).await,+ Some(SourceKind::GithubIssues) => sync_github_issues(state, &source, since).await,+ Some(SourceKind::JiraIssues) => sync_jira_issues(state, &source, since).await,Some(SourceKind::S3 | SourceKind::AzureBlob | SourceKind::Gcs) => {sync_object_storage(state, &source).await}@@ -729,7 +740,11 @@ async fn sync_rss(state: &AppState, source: &Source) -> anyhow::Result<SyncStats/// `doc_time` 取 `updated_at` 而不是 `created_at`:每次同步捕获的是"此刻这张/// 工单是什么样",认知时间该说这个状态是何时成立的。新增一条评论会改/// `updated_at`,于是内容变了、记一个新版本、`doc_time` 也跟着走。-async fn sync_github_issues(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {+async fn sync_github_issues(+ state: &AppState,+ source: &Source,+ since: Option<DateTime<Utc>>,+) -> anyhow::Result<SyncStats> {let repo = source.config["repo"].as_str().map(str::trim)@@ -762,7 +777,7 @@ async fn sync_github_issues(state: &AppState, source: &Source) -> anyhow::Result// 增量:GitHub 的 since 是"这之后更新过的"let mut issue_q: Vec<(&str, String)> = vec![("state", "all".into())];let mut comment_q: Vec<(&str, String)> = Vec::new();- if let Some(t) = source.last_sync_at {+ if let Some(t) = since {issue_q.push(("since", t.to_rfc3339()));comment_q.push(("since", t.to_rfc3339()));}@@ -822,7 +837,11 @@ async fn sync_github_issues(state: &AppState, source: &Source) -> anyhow::Result////// `doc_time` 取 `updated`,与 github_issues 同一口径:每次同步捕获的是/// "此刻这张工单是什么样",认知时间该说这个状态何时成立。-async fn sync_jira_issues(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {+async fn sync_jira_issues(+ state: &AppState,+ source: &Source,+ since: Option<DateTime<Utc>>,+) -> anyhow::Result<SyncStats> {let base_url = source.config["base_url"].as_str().map(str::trim)@@ -850,7 +869,7 @@ async fn sync_jira_issues(state: &AppState, source: &Source) -> anyhow::Result<S).await?;- let jql = crate::jira_issues::jql(project, source.last_sync_at);+ let jql = crate::jira_issues::jql(project, since);let (issues, total) = crate::jira_issues::fetch_all(&http, base_url, &jql, auth).await?;// **截断了就说出来。** 一个跑了多年的项目动辄上万张工单,翻页上限意味着// 这一轮只覆盖了一段;不报的话界面上"同步完成"就是一句误导@@ -904,18 +923,22 @@ fn slugify(title: &str) -> String {}/// 自定义拉取器 —— Utopia Ingest Interface:-/// `GET {endpoint}?since=<上次同步 RFC3339>`(首次同步不带 since;可配 Authorization 头),+/// `GET {endpoint}?since=<上次成功同步开始时间 RFC3339>`(首次同步不带 since;可配 Authorization 头),/// 响应 `{"items":[{"id":"稳定唯一ID","title":"文档名","content":"正文(纯文本/Markdown/HTML)",/// "doc_time":"RFC3339 可选","mime":"text/markdown 可选"}]}`。/// id → external_key(custom:{id}),三路判定生效:同 id 同内容跳过、新内容原地更新。-async fn sync_custom(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {+async fn sync_custom(+ state: &AppState,+ source: &Source,+ since: Option<DateTime<Utc>>,+) -> anyhow::Result<SyncStats> {let endpoint = source.config["endpoint"].as_str().filter(|s| !s.trim().is_empty()).ok_or_else(|| anyhow::anyhow!("custom source is missing config.endpoint"))?;let mut url =reqwest::Url::parse(endpoint).map_err(|e| anyhow::anyhow!("Invalid endpoint URL: {e}"))?;- if let Some(t) = source.last_sync_at {+ if let Some(t) = since {url.query_pairs_mut().append_pair("since", &t.to_rfc3339());}diff --git a/crates/utopia-server/src/source_checkpoint_tests.rs b/crates/utopia-server/src/source_checkpoint_tests.rsnew file mode 100644index 00000000..805e1211--- /dev/null+++ b/crates/utopia-server/src/source_checkpoint_tests.rs@@ -0,0 +1,137 @@+//! 增量游标代表已成功覆盖的窗口,失败的尝试不能把尚未读到的条目跳过去。+use super::sync_source;+use std::sync::Arc;+use uuid::Uuid;+use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate};++async fn retry_reads_the_uncovered_window(+ prior_success: bool,+ fail_before_retry: bool,+) -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = sqlx::PgPool::connect(&url).await?;+ let (org, ws, kb, source) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'checkpoint-test')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'checkpoint-test')")+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'checkpoint-test')",+ )+ .bind(kb)+ .bind(ws)+ .execute(&pool)+ .await?;+ let server = MockServer::start().await;+ sqlx::query(+ "INSERT INTO sources(id,kb_id,kind,name,config) VALUES($1,$2,'custom','fixture',$3)",+ )+ .bind(source)+ .bind(kb)+ .bind(serde_json::json!({"endpoint":server.uri()}))+ .execute(&pool)+ .await?;+ let started = chrono::Utc::now() - chrono::Duration::hours(2);+ let changed = started + chrono::Duration::minutes(30);+ if prior_success {+ let run = utopia_store::sources::start_run(&pool, source).await?;+ utopia_store::sources::finish_run(&pool, run, source, None, 0, 0).await?;+ sqlx::query("UPDATE source_sync_runs SET started_at=$2,finished_at=$3 WHERE id=$1")+ .bind(run)+ .bind(started)+ .bind(started + chrono::Duration::hours(1))+ .execute(&pool)+ .await?;+ utopia_store::sources::touch_sync_time(&pool, source, started + chrono::Duration::hours(1))+ .await?;+ }+ let dir = std::env::temp_dir().join(format!("utopia-checkpoint-{source}"));+ let cfg = utopia_core::config::AppConfig {+ data_dir: dir.to_string_lossy().into_owned(),+ ..Default::default()+ };+ let search = Arc::new(utopia_search::SearchIndex::open(&dir.join("search"))?);+ let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into());+ let result = async {+ if fail_before_retry {+ Mock::given(method("GET")).respond_with(ResponseTemplate::new(503)).mount(&server).await;+ assert!(sync_source(&state, source).await.is_err());+ let failed = utopia_store::sources::get(&pool, source).await?;+ assert_eq!(failed.last_sync_status, "failed");+ assert!(failed.last_sync_at.is_some(), "attempt time still drives scheduling and diagnostics");+ server.reset().await;+ }+ // **把真正发出去的下界记下来**:只断言那条漏掉的更新回来了是不够的,+ // 假如下界根本没发(每次都全量拉),这个断言照样过,而增量就悄悄没了+ let asked: std::sync::Arc<std::sync::Mutex<Vec<Option<chrono::DateTime<chrono::Utc>>>>> =+ Default::default();+ let seen = asked.clone();+ Mock::given(method("GET")).respond_with(move |request: &Request| {+ let since = request.url.query_pairs().find(|(k,_)| k == "since")+ .map(|(_,v)| chrono::DateTime::parse_from_rfc3339(&v).unwrap().with_timezone(&chrono::Utc));+ seen.lock().unwrap().push(since);+ let items = if since.is_none_or(|s| s <= changed) {+ serde_json::json!([{"id":"missed", "title":"Missed update", "content":"An update from the uncovered window"}])+ } else { serde_json::json!([]) };+ ResponseTemplate::new(200).set_body_json(serde_json::json!({"items":items}))+ }).mount(&server).await;+ sync_source(&state, source).await?;+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM documents WHERE source_id=$1")+ .bind(source).fetch_one(&pool).await?;+ anyhow::ensure!(count == 1, "incremental cursor skipped an unimported update: {count} documents");+ let ok = utopia_store::sources::get(&pool, source).await?;+ assert_eq!(ok.last_sync_status, "ok");+ // 下界取自上一次**成功**那一轮的开始,不是上一次尝试的结束,也不是没有下界+ let asked = asked.lock().unwrap().clone();+ let last = asked.last().copied().flatten();+ if prior_success {+ let last = last.expect("没有带下界:增量拉取整个没了");+ assert!(+ (last - started).num_seconds().abs() <= 1,+ "下界应当是上一次成功那轮的开始 {started},实得 {last}"+ );+ assert!(last < changed, "下界晚于那次改动,漏掉的窗口又被跳过了");+ } else {+ assert!(last.is_none(), "没有成功过的源不该带下界,实得 {last:?}");+ }+ Ok::<_, anyhow::Error>(())+ }.await;+ sqlx::query("DELETE FROM knowledge_bases WHERE id=$1")+ .bind(kb)+ .execute(&pool)+ .await?;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ drop(state);+ let _ = std::fs::remove_dir_all(dir);+ result+}++#[tokio::test]+async fn a_failed_first_sync_does_not_skip_the_existing_items() -> anyhow::Result<()> {+ retry_reads_the_uncovered_window(false, true).await+}++#[tokio::test]+async fn retry_keeps_updates_from_the_previous_successful_run_window() -> anyhow::Result<()> {+ retry_reads_the_uncovered_window(true, true).await+}++#[tokio::test]+async fn updates_during_a_successful_sync_remain_in_the_next_window() -> anyhow::Result<()> {+ retry_reads_the_uncovered_window(true, false).await+}diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rsindex 18c29efe..b5780530 100644--- a/crates/utopia-store/src/sources.rs+++ b/crates/utopia-store/src/sources.rs@@ -475,6 +475,20 @@ pub async fn touch_sync_time(pool: &PgPool, id: Uuid, at: DateTime<Utc>) -> AppR// 同步运行记录(渠道审计历史)// ---------------------------------------------------------------------------+/// 调度时钟包含失败尝试,不能作增量游标。回到上次成功运行的开始,+/// 让那次拉取期间发生的更新也能在下一轮读到;没有成功记录就重新全量读取。+pub async fn last_successful_sync_start(+ pool: &PgPool,+ source_id: Uuid,+) -> AppResult<Option<DateTime<Utc>>> {+ Ok(sqlx::query_scalar(+ "SELECT max(started_at) FROM source_sync_runs WHERE source_id = $1 AND status = 'ok'",+ )+ .bind(source_id)+ .fetch_one(pool)+ .await?)+}+pub async fn start_run(pool: &PgPool, source_id: Uuid) -> AppResult<Uuid> {let id = Uuid::now_v7();sqlx::query("INSERT INTO source_sync_runs (id, source_id) VALUES ($1, $2)")