Skip to content

World's first open-source enterprise world model.

Keep failed sync attempts from skipping incremental source updates (#794)

Maya Kid84920bdf38c4
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs
index dda5a51d..ab34d192 100644
--- a/crates/utopia-server/src/ingest_sources.rs
+++ b/crates/utopia-server/src/ingest_sources.rs
@@ -11,6 +11,10 @@ use utopia_core::models::Source;
use utopia_core::models::SourceKind;
use uuid::Uuid;
+#[cfg(test)]
+#[path = "source_checkpoint_tests.rs"]
+mod source_checkpoint_tests;
+
/// 单次同步的新文档上限(防超长 feed/URL 列表拖垮任务)
const MAX_NEW_PER_SYNC: usize = 200;
const MAX_FEED_BYTES: usize = 4 * 1024 * 1024;
@@ -62,17 +66,24 @@ impl SyncStats {
pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<()> {
let source = utopia_store::sources::get(&state.pool, source_id).await?;
+ let kind = SourceKind::parse(&source.kind);
+ let since = match kind {
+ Some(SourceKind::Custom | SourceKind::GithubIssues | SourceKind::JiraIssues) => {
+ utopia_store::sources::last_successful_sync_start(&state.pool, source_id).await?
+ }
+ _ => None,
+ };
utopia_store::sources::mark_running(&state.pool, source_id).await?;
let run_id = utopia_store::sources::start_run(&state.pool, source_id).await?;
state.emit_source(source.kb_id);
// 按枚举穷举:加一种来源就得在这里决定它怎么同步,编译器不放过漏掉的那一支
- let outcome = match SourceKind::parse(&source.kind) {
+ let outcome = match kind {
Some(SourceKind::Url) => sync_urls(state, &source).await,
Some(SourceKind::Rss) => sync_rss(state, &source).await,
- Some(SourceKind::Custom) => sync_custom(state, &source).await,
- Some(SourceKind::GithubIssues) => sync_github_issues(state, &source).await,
- Some(SourceKind::JiraIssues) => sync_jira_issues(state, &source).await,
+ Some(SourceKind::Custom) => sync_custom(state, &source, since).await,
+ Some(SourceKind::GithubIssues) => sync_github_issues(state, &source, since).await,
+ Some(SourceKind::JiraIssues) => sync_jira_issues(state, &source, since).await,
Some(SourceKind::S3 | SourceKind::AzureBlob | SourceKind::Gcs) => {
sync_object_storage(state, &source).await
}
@@ -729,7 +740,11 @@ async fn sync_rss(state: &AppState, source: &Source) -> anyhow::Result<SyncStats
/// `doc_time` 取 `updated_at` 而不是 `created_at`:每次同步捕获的是"此刻这张
/// 工单是什么样",认知时间该说这个状态是何时成立的。新增一条评论会改
/// `updated_at`,于是内容变了、记一个新版本、`doc_time` 也跟着走。
-async fn sync_github_issues(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {
+async fn sync_github_issues(
+ state: &AppState,
+ source: &Source,
+ since: Option<DateTime<Utc>>,
+) -> anyhow::Result<SyncStats> {
let repo = source.config["repo"]
.as_str()
.map(str::trim)
@@ -762,7 +777,7 @@ async fn sync_github_issues(state: &AppState, source: &Source) -> anyhow::Result
// 增量:GitHub 的 since 是"这之后更新过的"
let mut issue_q: Vec<(&str, String)> = vec![("state", "all".into())];
let mut comment_q: Vec<(&str, String)> = Vec::new();
- if let Some(t) = source.last_sync_at {
+ if let Some(t) = since {
issue_q.push(("since", t.to_rfc3339()));
comment_q.push(("since", t.to_rfc3339()));
}
@@ -822,7 +837,11 @@ async fn sync_github_issues(state: &AppState, source: &Source) -> anyhow::Result
///
/// `doc_time` 取 `updated`,与 github_issues 同一口径:每次同步捕获的是
/// "此刻这张工单是什么样",认知时间该说这个状态何时成立。
-async fn sync_jira_issues(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {
+async fn sync_jira_issues(
+ state: &AppState,
+ source: &Source,
+ since: Option<DateTime<Utc>>,
+) -> anyhow::Result<SyncStats> {
let base_url = source.config["base_url"]
.as_str()
.map(str::trim)
@@ -850,7 +869,7 @@ async fn sync_jira_issues(state: &AppState, source: &Source) -> anyhow::Result<S
)
.await?;
- let jql = crate::jira_issues::jql(project, source.last_sync_at);
+ let jql = crate::jira_issues::jql(project, since);
let (issues, total) = crate::jira_issues::fetch_all(&http, base_url, &jql, auth).await?;
// **截断了就说出来。** 一个跑了多年的项目动辄上万张工单,翻页上限意味着
// 这一轮只覆盖了一段;不报的话界面上"同步完成"就是一句误导
@@ -904,18 +923,22 @@ fn slugify(title: &str) -> String {
}
/// 自定义拉取器 —— Utopia Ingest Interface:
-/// `GET {endpoint}?since=<上次同步 RFC3339>`(首次同步不带 since;可配 Authorization 头),
+/// `GET {endpoint}?since=<上次成功同步开始时间 RFC3339>`(首次同步不带 since;可配 Authorization 头),
/// 响应 `{"items":[{"id":"稳定唯一ID","title":"文档名","content":"正文(纯文本/Markdown/HTML)",
/// "doc_time":"RFC3339 可选","mime":"text/markdown 可选"}]}`。
/// id → external_key(custom:{id}),三路判定生效:同 id 同内容跳过、新内容原地更新。
-async fn sync_custom(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {
+async fn sync_custom(
+ state: &AppState,
+ source: &Source,
+ since: Option<DateTime<Utc>>,
+) -> anyhow::Result<SyncStats> {
let endpoint = source.config["endpoint"]
.as_str()
.filter(|s| !s.trim().is_empty())
.ok_or_else(|| anyhow::anyhow!("custom source is missing config.endpoint"))?;
let mut url =
reqwest::Url::parse(endpoint).map_err(|e| anyhow::anyhow!("Invalid endpoint URL: {e}"))?;
- if let Some(t) = source.last_sync_at {
+ if let Some(t) = since {
url.query_pairs_mut().append_pair("since", &t.to_rfc3339());
}
diff --git a/crates/utopia-server/src/source_checkpoint_tests.rs b/crates/utopia-server/src/source_checkpoint_tests.rs
new file mode 100644
index 00000000..805e1211
--- /dev/null
+++ b/crates/utopia-server/src/source_checkpoint_tests.rs
@@ -0,0 +1,137 @@
+//! 增量游标代表已成功覆盖的窗口,失败的尝试不能把尚未读到的条目跳过去。
+use super::sync_source;
+use std::sync::Arc;
+use uuid::Uuid;
+use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate};
+
+async fn retry_reads_the_uncovered_window(
+ prior_success: bool,
+ fail_before_retry: bool,
+) -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = sqlx::PgPool::connect(&url).await?;
+ let (org, ws, kb, source) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'checkpoint-test')")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'checkpoint-test')")
+ .bind(ws)
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'checkpoint-test')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(&pool)
+ .await?;
+ let server = MockServer::start().await;
+ sqlx::query(
+ "INSERT INTO sources(id,kb_id,kind,name,config) VALUES($1,$2,'custom','fixture',$3)",
+ )
+ .bind(source)
+ .bind(kb)
+ .bind(serde_json::json!({"endpoint":server.uri()}))
+ .execute(&pool)
+ .await?;
+ let started = chrono::Utc::now() - chrono::Duration::hours(2);
+ let changed = started + chrono::Duration::minutes(30);
+ if prior_success {
+ let run = utopia_store::sources::start_run(&pool, source).await?;
+ utopia_store::sources::finish_run(&pool, run, source, None, 0, 0).await?;
+ sqlx::query("UPDATE source_sync_runs SET started_at=$2,finished_at=$3 WHERE id=$1")
+ .bind(run)
+ .bind(started)
+ .bind(started + chrono::Duration::hours(1))
+ .execute(&pool)
+ .await?;
+ utopia_store::sources::touch_sync_time(&pool, source, started + chrono::Duration::hours(1))
+ .await?;
+ }
+ let dir = std::env::temp_dir().join(format!("utopia-checkpoint-{source}"));
+ let cfg = utopia_core::config::AppConfig {
+ data_dir: dir.to_string_lossy().into_owned(),
+ ..Default::default()
+ };
+ let search = Arc::new(utopia_search::SearchIndex::open(&dir.join("search"))?);
+ let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into());
+ let result = async {
+ if fail_before_retry {
+ Mock::given(method("GET")).respond_with(ResponseTemplate::new(503)).mount(&server).await;
+ assert!(sync_source(&state, source).await.is_err());
+ let failed = utopia_store::sources::get(&pool, source).await?;
+ assert_eq!(failed.last_sync_status, "failed");
+ assert!(failed.last_sync_at.is_some(), "attempt time still drives scheduling and diagnostics");
+ server.reset().await;
+ }
+ // **把真正发出去的下界记下来**:只断言那条漏掉的更新回来了是不够的,
+ // 假如下界根本没发(每次都全量拉),这个断言照样过,而增量就悄悄没了
+ let asked: std::sync::Arc<std::sync::Mutex<Vec<Option<chrono::DateTime<chrono::Utc>>>>> =
+ Default::default();
+ let seen = asked.clone();
+ Mock::given(method("GET")).respond_with(move |request: &Request| {
+ let since = request.url.query_pairs().find(|(k,_)| k == "since")
+ .map(|(_,v)| chrono::DateTime::parse_from_rfc3339(&v).unwrap().with_timezone(&chrono::Utc));
+ seen.lock().unwrap().push(since);
+ let items = if since.is_none_or(|s| s <= changed) {
+ serde_json::json!([{"id":"missed", "title":"Missed update", "content":"An update from the uncovered window"}])
+ } else { serde_json::json!([]) };
+ ResponseTemplate::new(200).set_body_json(serde_json::json!({"items":items}))
+ }).mount(&server).await;
+ sync_source(&state, source).await?;
+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM documents WHERE source_id=$1")
+ .bind(source).fetch_one(&pool).await?;
+ anyhow::ensure!(count == 1, "incremental cursor skipped an unimported update: {count} documents");
+ let ok = utopia_store::sources::get(&pool, source).await?;
+ assert_eq!(ok.last_sync_status, "ok");
+ // 下界取自上一次**成功**那一轮的开始,不是上一次尝试的结束,也不是没有下界
+ let asked = asked.lock().unwrap().clone();
+ let last = asked.last().copied().flatten();
+ if prior_success {
+ let last = last.expect("没有带下界:增量拉取整个没了");
+ assert!(
+ (last - started).num_seconds().abs() <= 1,
+ "下界应当是上一次成功那轮的开始 {started},实得 {last}"
+ );
+ assert!(last < changed, "下界晚于那次改动,漏掉的窗口又被跳过了");
+ } else {
+ assert!(last.is_none(), "没有成功过的源不该带下界,实得 {last:?}");
+ }
+ Ok::<_, anyhow::Error>(())
+ }.await;
+ sqlx::query("DELETE FROM knowledge_bases WHERE id=$1")
+ .bind(kb)
+ .execute(&pool)
+ .await?;
+ sqlx::query("DELETE FROM organizations WHERE id=$1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ drop(state);
+ let _ = std::fs::remove_dir_all(dir);
+ result
+}
+
+#[tokio::test]
+async fn a_failed_first_sync_does_not_skip_the_existing_items() -> anyhow::Result<()> {
+ retry_reads_the_uncovered_window(false, true).await
+}
+
+#[tokio::test]
+async fn retry_keeps_updates_from_the_previous_successful_run_window() -> anyhow::Result<()> {
+ retry_reads_the_uncovered_window(true, true).await
+}
+
+#[tokio::test]
+async fn updates_during_a_successful_sync_remain_in_the_next_window() -> anyhow::Result<()> {
+ retry_reads_the_uncovered_window(true, false).await
+}
diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rs
index 18c29efe..b5780530 100644
--- a/crates/utopia-store/src/sources.rs
+++ b/crates/utopia-store/src/sources.rs
@@ -475,6 +475,20 @@ pub async fn touch_sync_time(pool: &PgPool, id: Uuid, at: DateTime<Utc>) -> AppR
// 同步运行记录(渠道审计历史)
// ---------------------------------------------------------------------------
+/// 调度时钟包含失败尝试,不能作增量游标。回到上次成功运行的开始,
+/// 让那次拉取期间发生的更新也能在下一轮读到;没有成功记录就重新全量读取。
+pub async fn last_successful_sync_start(
+ pool: &PgPool,
+ source_id: Uuid,
+) -> AppResult<Option<DateTime<Utc>>> {
+ Ok(sqlx::query_scalar(
+ "SELECT max(started_at) FROM source_sync_runs WHERE source_id = $1 AND status = 'ok'",
+ )
+ .bind(source_id)
+ .fetch_one(pool)
+ .await?)
+}
+
pub async fn start_run(pool: &PgPool, source_id: Uuid) -> AppResult<Uuid> {
let id = Uuid::now_v7();
sqlx::query("INSERT INTO source_sync_runs (id, source_id) VALUES ($1, $2)")