World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Preserve unchanged issue comments when incremental GitHub sync replaces a document (#1061)
Maya Kid3410829d5c0a
diff --git a/crates/utopia-server/src/github_issues.rs b/crates/utopia-server/src/github_issues.rsindex aabccbff..dffe3e66 100644--- a/crates/utopia-server/src/github_issues.rs+++ b/crates/utopia-server/src/github_issues.rs@@ -9,25 +9,26 @@//! 与我们给维基百科历史快照做的是同一个判断:**别取现在,取变化**。区别是//! GitHub 直接把变化给了你,不用像维基那样从修订列表里采样。//!-//! ## 评论走仓库级,事件走逐个——这不是不一致,是两个端点能力不同+//! ## 工单列表增量筛选,评论与事件逐工单读取//!//! 第一版想让三样都走仓库级端点、一次分页取全,避免 200 张工单 401 次请求//! (未认证时 GitHub 每小时只给 60 次)。**拿真实数据一跑就发现事件那一路是错的**://!-//! - `issues/comments` 支持 `since`,增量窗口内的评论一次取全。**好用**。+//! - `issues/comments` 支持 `since`,但只给改动过的评论。文档是整篇替换,+//! 因而只改标题也会把所有旧评论从当前文档删掉;新增评论则只留下新增的部分。//! - `issues/events` **不支持 `since`**,只能从最新往回翻。而 GitHub 的模型里//! PR 也产生 issue 事件——实测本仓库工单事件埋在第 5 页,换一个 PR 活跃的仓库//! 就会被推到翻页上限之外。**于是"状态变更史"悄悄变成空的,而它正是这个来源//! 存在的理由。**//!-//! 所以事件改成逐工单取 `GET /repos/{repo}/issues/{n}/events`。N+1 的代价是真的,-//! 但 N 只是**本轮要写入的工单数**:首次同步等于工单总数,之后有 `since` 兜着,-//! 通常是个位数。用一次准确换一次省事,这里该换。+//! 所以评论与事件都逐工单取,不带 `since`。只对筛选后本轮要写入的工单请求,+//! 避免全仓库的无关评论消耗分页预算。请求数增加的代价是真的,但 N 只是+//! **本轮要写入的工单数**:之后有工单列表的 `since` 兜着,通常是个位数。//!//! 三次拉取因此是://!//! - `GET /repos/{repo}/issues?state=all&since=` —— 工单本体(分页)-//! - `GET /repos/{repo}/issues/comments?since=` —— 全仓库评论(分页),按号归拢+//! - `GET /repos/{repo}/issues/{n}/comments` —— 每张工单的评论(分页)//! - `GET /repos/{repo}/issues/{n}/events` —— 每张工单一次//!//! ## 关于 PR@@ -39,10 +40,9 @@use crate::time_text;use chrono::{DateTime, Utc};use serde::Deserialize;-use std::collections::HashMap;/// 一次同步最多取多少页(每页 100)。GitHub 的分页没有天然终点,-/// 一个活跃仓库能翻很久;这里封顶,超出的等下一次 `since` 增量取。+/// 一个活跃仓库能翻很久;这里沿用单端点的分页预算。const MAX_PAGES: u32 = 10;const PER_PAGE: u32 = 100;@@ -192,28 +192,14 @@ pub fn render(issue: &Issue, comments: &[&Comment], events: &[&Event]) -> Stringout}-/// 把全仓库的评论按工单号归拢,各自按时间升序。-///-/// 评论是**全仓库**取回来的,里面混着不在本次工单集合里的(增量窗口不同步)。-/// 归不到工单上的直接丢——它们下次会跟着自己的工单一起回来。-pub fn group_comments<'a>(- issues: &'a [Issue],- comments: &'a [Comment],-) -> Vec<(&'a Issue, Vec<&'a Comment>)> {- let mut by_issue: HashMap<i64, Vec<&Comment>> = HashMap::new();- for c in comments {- if let Some(n) = issue_number_from_url(&c.issue_url) {- by_issue.entry(n).or_default().push(c);- }- }- issues+/// 只保留这张工单的评论,按时间升序;响应里归不到它的评论不能混进正文。+pub fn comments_for_issue<'a>(issue: &Issue, comments: &'a [Comment]) -> Vec<&'a Comment> {+ let mut comments: Vec<_> = comments.iter()- .map(|issue| {- let mut cs = by_issue.remove(&issue.number).unwrap_or_default();- cs.sort_by_key(|c| c.created_at);- (issue, cs)- })- .collect()+ .filter(|comment| issue_number_from_url(&comment.issue_url) == Some(issue.number))+ .collect();+ comments.sort_by_key(|comment| comment.created_at);+ comments}/// 事件按时间升序。逐工单端点返回的**看起来**是升序,但顺序不是契约,@@ -358,11 +344,11 @@ mod tests {assert_eq!(at, i.created_at);}- /// 评论是**全仓库**取的,归拢必须按工单号,且按时间升序。+ /// 评论仍按工单号验证,且按时间升序。/// 归错了的后果很隐蔽:另一张工单的讨论出现在这张的正文里。#[test]- fn repo_wide_comments_land_on_the_right_issue() {- let issues = vec![+ fn comments_belong_to_the_issue_and_are_chronological() {+ let issues = [issue(serde_json::json!({"number": 1, "title": "one", "state": "open", "body": null,"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z",@@ -381,19 +367,16 @@ mod tests {"user": {"login": "y"}, "created_at": "2026-01-03T00:00:00Z", "body": "第一条"},// 不在本次工单集合里:该被丢掉,而不是挂到别人身上{"issue_url": "https://api.github.com/repos/a/b/issues/99",- "user": {"login": "z"}, "created_at": "2026-01-04T00:00:00Z", "body": "别人的"}+ "user": {"login": "z"}, "created_at": "2026-01-04T00:00:00Z", "body": "别人的"},+ {"issue_url": "nonsense",+ "user": null, "created_at": "2026-01-02T00:00:00Z", "body": "无法归属"}])).unwrap();- let grouped = group_comments(&issues, &comments);- assert_eq!(grouped.len(), 2);-- let (one, one_comments) = &grouped[0];- assert_eq!(one.number, 1);+ let one_comments = comments_for_issue(&issues[0], &comments);assert!(one_comments.is_empty(), "1 号没有评论");- let (two, two_comments) = &grouped[1];- assert_eq!(two.number, 2);- assert_eq!(two_comments.len(), 2, "99 号那条不该混进来");+ let two_comments = comments_for_issue(&issues[1], &comments);+ assert_eq!(two_comments.len(), 2, "其他工单和无法归属的评论不该混进来");// 升序:先"第一条"(01-03)后"第二条"(01-05)assert_eq!(two_comments[0].body.as_deref(), Some("第一条"));}@@ -459,11 +442,9 @@ mod tests {"夹具应当只含真工单");- let grouped = group_comments(&f.issues, &f.comments);- assert_eq!(grouped.len(), f.issues.len());-// 每张工单都排得出一篇非空文档,且抬头那句带真实日期- for (issue, cs) in &grouped {+ for issue in &f.issues {+ let cs = comments_for_issue(issue, &f.comments);let events = sort_events(f.events_by_issue.get(&issue.number.to_string())@@ -471,7 +452,7 @@ mod tests {.unwrap_or_default(),);let es: Vec<&Event> = events.iter().collect();- let doc = render(issue, cs, &es);+ let doc = render(issue, &cs, &es);assert!(doc.contains(&format!("# #{} ", issue.number)),"#{} 的抬头不对:{doc}",@@ -486,7 +467,8 @@ mod tests {// 这份夹具里每张工单都被关掉过,所以历史一节必须出现——// 它是这个来源存在的理由,空了就等于退回"抓一个网页"- let (first, cs) = &grouped[0];+ let first = &f.issues[0];+ let cs = comments_for_issue(first, &f.comments);let events = sort_events(f.events_by_issue.get(&first.number.to_string())@@ -495,7 +477,7 @@ mod tests {);assert!(!events.is_empty(), "夹具里 #{} 没有事件", first.number);let es: Vec<&Event> = events.iter().collect();- let doc = render(first, cs, &es);+ let doc = render(first, &cs, &es);assert!(doc.contains("## History"), "历史一节缺失:{doc}");assert!(doc.contains("— closed by"), "关闭事件没写进历史:{doc}");}diff --git a/crates/utopia-server/src/github_sync_tests.rs b/crates/utopia-server/src/github_sync_tests.rsnew file mode 100644index 00000000..39b38db2--- /dev/null+++ b/crates/utopia-server/src/github_sync_tests.rs@@ -0,0 +1,291 @@+//! A replacement issue document must contain its entire discussion, even when only+//! its title changed since the last sync. These fixtures exercise real persistence.+use super::sync_github_issues_with_client;+use chrono::{DateTime, Utc};+use serde_json::{json, Value};+use std::sync::{Arc, Mutex};+use utopia_core::models::{Document, Source};+use uuid::Uuid;+use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate};++#[derive(Clone, Copy, PartialEq, Eq)]+enum Stage {+ Initial,+ TitleOnly,+ DiscussionUpdated,+ CommentPageFailure,+ Repeated,+ CommentsDeleted,+}++fn time(value: &str) -> DateTime<Utc> {+ value.parse().unwrap()+}++fn issue(number: i64, stage: Stage) -> Value {+ json!({+ "number": number,+ "title": if stage == Stage::Initial { "Original issue" } else { "Updated issue" },+ "state": "open",+ "body": "A synthetic discussion for the sync regression.",+ "created_at": "2026-01-01T00:00:00Z",+ "updated_at": if stage == Stage::Initial { "2026-01-02T00:00:00Z" } else { "2026-01-04T00:00:00Z" },+ "pull_request": if number == 20 { json!({"url":"https://api.github.com/repos/test/repo/pulls/20"}) } else { Value::Null },+ })+}++fn comments(stage: Stage) -> Vec<Value> {+ if stage == Stage::CommentsDeleted {+ return vec![];+ }+ (0..if matches!(stage, Stage::Initial | Stage::TitleOnly) { 100 } else { 101 })+ .map(|index| {+ let created = time("2026-01-01T01:00:00Z") + chrono::Duration::seconds(index);+ json!({+ "issue_url": "https://api.github.com/repos/test/repo/issues/18",+ "user": {"login":"fixture-author"},+ "created_at": if index == 100 { time("2026-01-04T00:00:00Z") } else { created },+ "body": match index {+ 0 => "The original conclusion must survive every refresh.".to_owned(),+ 1 if matches!(stage, Stage::Initial | Stage::TitleOnly) => "An explanation before correction.".to_owned(),+ 1 => "The corrected explanation.".to_owned(),+ 100 => "A new follow-up on the second page.".to_owned(),+ _ => format!("Earlier discussion comment {index}."),+ }+ })+ })+ .collect()+}++async fn document(+ state: &crate::state::AppState,+ source: &Source,+) -> anyhow::Result<(Document, String)> {+ let doc = utopia_store::documents::find_by_external_key(+ &state.pool,+ source.id,+ "github:test/repo#18",+ )+ .await?+ .expect("the issue should have been persisted");+ let body = String::from_utf8(state.blob.get(&doc.sha256).await?)?;+ Ok((doc, body))+}++async fn versions(state: &crate::state::AppState, document_id: Uuid) -> anyhow::Result<i64> {+ Ok(+ sqlx::query_scalar("SELECT count(*) FROM document_versions WHERE document_id=$1")+ .bind(document_id)+ .fetch_one(&state.pool)+ .await?,+ )+}++#[tokio::test]+async fn incremental_github_sync_preserves_the_complete_discussion() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = sqlx::PgPool::connect(&url).await?;+ let (org, ws, kb, source_id) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'github-sync-test')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'github-sync-test')")+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'github-sync-test')",+ )+ .bind(kb)+ .bind(ws)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO sources(id,kb_id,kind,name,config) VALUES($1,$2,'github_issues','fixture',$3)",+ )+ .bind(source_id)+ .bind(kb)+ .bind(json!({"repo":"test/repo", "auth_header":"Bearer fixture-token"}))+ .execute(&pool)+ .await?;+ let source = utopia_store::sources::get(&pool, source_id).await?;+ let dir = tempfile::tempdir()?;+ let cfg = utopia_core::config::AppConfig {+ data_dir: dir.path().to_string_lossy().into_owned(),+ ..Default::default()+ };+ let search = Arc::new(utopia_search::SearchIndex::open(+ &dir.path().join("search"),+ )?);+ let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into());+ let server = MockServer::start().await;+ let stage = Arc::new(Mutex::new(Stage::Initial));+ let current = stage.clone();+ Mock::given(method("GET"))+ .respond_with(move |request: &Request| {+ let stage = *current.lock().unwrap();+ let page = request+ .url+ .query_pairs()+ .find(|(key, _)| key == "page")+ .map(|(_, value)| value.parse::<usize>().unwrap())+ .unwrap_or(1);+ let since = request.url.query_pairs().any(|(key, _)| key == "since");+ let path = request.url.path();+ let rows = match path {+ "/repos/test/repo/issues" => {+ if since {+ vec![issue(18, stage), issue(20, stage)]+ } else {+ vec![issue(18, stage), issue(19, stage), issue(20, stage)]+ }+ }+ // Keep the old endpoint realistic, so the regression fails on lost+ // stored history rather than on an unexpected request path.+ "/repos/test/repo/issues/comments" | "/repos/test/repo/issues/18/comments" => {+ if stage == Stage::CommentPageFailure && page == 2 {+ return ResponseTemplate::new(503);+ }+ comments(stage)+ .into_iter()+ .enumerate()+ .filter(|(index, _)| {+ !since+ || (!matches!(stage, Stage::Initial | Stage::TitleOnly)+ && (*index == 1 || *index == 100))+ })+ .map(|(_, value)| value)+ .collect()+ }+ "/repos/test/repo/issues/19/comments" => vec![],+ "/repos/test/repo/issues/18/events" | "/repos/test/repo/issues/19/events" => vec![],+ _ => return ResponseTemplate::new(404),+ };+ let rows: Vec<_> = rows.into_iter().skip((page - 1) * 100).take(100).collect();+ ResponseTemplate::new(200).set_body_json(rows)+ })+ .mount(&server)+ .await;+ let http = reqwest::Client::builder().no_proxy().build()?;+ let since = Some(time("2026-01-03T00:00:00Z"));+ let result = async {+ let first =+ sync_github_issues_with_client(&state, &source, None, &http, &server.uri()).await?;+ anyhow::ensure!(+ first.created == 2,+ "both issues, including the one without comments, should be created"+ );+ let (original, original_body) = document(&state, &source).await?;+ anyhow::ensure!(original_body.contains("The original conclusion"));++ *stage.lock().unwrap() = Stage::TitleOnly;+ let title_only =+ sync_github_issues_with_client(&state, &source, since, &http, &server.uri()).await?;+ anyhow::ensure!(title_only.updated == 1);+ let (_, title_body) = document(&state, &source).await?;+ anyhow::ensure!(+ title_body.contains("The original conclusion")+ && title_body.contains("An explanation before correction."),+ "updating only an issue title erased its stored discussion"+ );++ *stage.lock().unwrap() = Stage::DiscussionUpdated;+ let changed =+ sync_github_issues_with_client(&state, &source, since, &http, &server.uri()).await?;+ anyhow::ensure!(changed.updated == 1 && changed.created == 0);+ let (updated, body) = document(&state, &source).await?;+ anyhow::ensure!(+ body.contains("The original conclusion"),+ "incremental sync erased an unchanged comment from the stored document"+ );+ anyhow::ensure!(body.contains("The corrected explanation."));+ anyhow::ensure!(!body.contains("An explanation before correction."));+ anyhow::ensure!(body.contains("A new follow-up on the second page."));+ anyhow::ensure!(updated.id == original.id && updated.sha256 != original.sha256);+ anyhow::ensure!(updated.doc_time == Some(time("2026-01-04T00:00:00Z")));+ anyhow::ensure!(versions(&state, updated.id).await? == 3);+ anyhow::ensure!(+ String::from_utf8(state.blob.get(&original.sha256).await?)? == original_body+ );++ *stage.lock().unwrap() = Stage::CommentPageFailure;+ let failed =+ sync_github_issues_with_client(&state, &source, since, &http, &server.uri()).await;+ anyhow::ensure!(failed.is_err(), "a failed comment page must fail the sync");+ let (after_failure, after_failure_body) = document(&state, &source).await?;+ anyhow::ensure!(after_failure.sha256 == updated.sha256 && after_failure_body == body);+ anyhow::ensure!(versions(&state, updated.id).await? == 3);++ *stage.lock().unwrap() = Stage::Repeated;+ let repeated =+ sync_github_issues_with_client(&state, &source, since, &http, &server.uri()).await?;+ anyhow::ensure!(repeated.created == 0 && repeated.updated == 0);+ anyhow::ensure!(versions(&state, updated.id).await? == 3);++ // An actually empty current discussion must remove deleted comments;+ // concatenating the previous document would preserve stale content.+ *stage.lock().unwrap() = Stage::CommentsDeleted;+ let cleared =+ sync_github_issues_with_client(&state, &source, since, &http, &server.uri()).await?;+ anyhow::ensure!(cleared.updated == 1);+ let (after_clear, clear_body) = document(&state, &source).await?;+ anyhow::ensure!(after_clear.id == original.id && clear_body.contains("Updated issue"));+ anyhow::ensure!(+ !clear_body.contains("The original conclusion")+ && !clear_body.contains("The corrected explanation")+ );+ anyhow::ensure!(versions(&state, updated.id).await? == 4);++ let requests = server.received_requests().await.unwrap();+ anyhow::ensure!(requests.iter().all(|request| request+ .headers+ .get("authorization")+ .is_some_and(|value| value == "Bearer fixture-token")));+ anyhow::ensure!(+ !requests+ .iter()+ .any(|request| request.url.path().contains("/issues/20/")),+ "excluded pull requests must not consume comment or event requests"+ );+ let issue_requests: Vec<_> = requests+ .iter()+ .filter(|request| request.url.path() == "/repos/test/repo/issues")+ .collect();+ anyhow::ensure!(issue_requests.len() == 6);+ for (index, request) in issue_requests.iter().enumerate() {+ let lower_bound = request+ .url+ .query_pairs()+ .find(|(key, _)| key == "since")+ .map(|(_, value)| time(&value));+ anyhow::ensure!(lower_bound == if index == 0 { None } else { since });+ }+ anyhow::ensure!(+ requests+ .iter()+ .filter(|request| request.url.path().ends_with("/comments"))+ .all(|request| !request.url.query_pairs().any(|(key, _)| key == "since")),+ "comments must be complete even while the issue list remains incremental"+ );+ Ok::<_, anyhow::Error>(())+ }+ .await;+ sqlx::query("DELETE FROM knowledge_bases WHERE id=$1")+ .bind(kb)+ .execute(&pool)+ .await?;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ result+}diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rsindex 2040b996..b0cedc1d 100644--- a/crates/utopia-server/src/ingest_sources.rs+++ b/crates/utopia-server/src/ingest_sources.rs@@ -737,9 +737,9 @@ async fn sync_rss(state: &AppState, source: &Source) -> anyhow::Result<SyncStats/// GitHub 工单:一张工单一篇文档,正文里带它的状态变更史。///-/// 工单与评论走**仓库级 + `since`**(一次分页取全),事件走**逐工单**——-/// 不是不一致,是 `issues/events` 不支持 `since` 且会被 PR 事件淹没,-/// 拿真实仓库一跑就发现状态变更史悄悄空了。详见 [`crate::github_issues`]。+/// 工单列表按 `since` 增量筛选,评论与事件逐工单读取,不带增量窗口。+/// 文档每次整篇替换:评论也按 `since` 过滤会把未改动的旧讨论删掉。+/// 详见 [`crate::github_issues`]。////// `doc_time` 取 `updated_at` 而不是 `created_at`:每次同步捕获的是"此刻这张/// 工单是什么样",认知时间该说这个状态是何时成立的。新增一条评论会改@@ -748,6 +748,24 @@ async fn sync_github_issues(state: &AppState,source: &Source,since: Option<DateTime<Utc>>,+) -> anyhow::Result<SyncStats> {+ // api.github.com 是固定的公网主机,所以按内容级的严格度收(`Reach::Content`)+ let http = crate::http_fetch::client_for(+ &reqwest::Url::parse("https://api.github.com/")?,+ crate::http_fetch::Reach::Content,+ crate::http_fetch::Limits::default(),+ )+ .await?;+ sync_github_issues_with_client(state, source, since, &http, "https://api.github.com").await+}++// HTTP 接缝让回归测试走同一条同步与落库路径,生产仍固定访问 GitHub 公网 API。+async fn sync_github_issues_with_client(+ state: &AppState,+ source: &Source,+ since: Option<DateTime<Utc>>,+ http: &reqwest::Client,+ api_base: &str,) -> anyhow::Result<SyncStats> {let repo = source.config["repo"].as_str()@@ -769,43 +787,37 @@ async fn sync_github_issues(.as_bool().unwrap_or(false);- // api.github.com 是固定的公网主机,所以按内容级的严格度收(`Reach::Content`)- let http = crate::http_fetch::client_for(- &reqwest::Url::parse("https://api.github.com/")?,- crate::http_fetch::Reach::Content,- crate::http_fetch::Limits::default(),- )- .await?;- let base = format!("https://api.github.com/repos/{repo}");+ let base = format!("{api_base}/repos/{repo}");// 增量:GitHub 的 since 是"这之后更新过的"let mut issue_q: Vec<(&str, String)> = vec![("state", "all".into())];- let mut comment_q: Vec<(&str, String)> = Vec::new();if let Some(t) = since {issue_q.push(("since", t.to_rfc3339()));- comment_q.push(("since", t.to_rfc3339()));}let issues: Vec<crate::github_issues::Issue> =- crate::github_issues::fetch_all(&http, &format!("{base}/issues"), &issue_q, auth).await?;- let comments: Vec<crate::github_issues::Comment> = crate::github_issues::fetch_all(- &http,- &format!("{base}/issues/comments"),- &comment_q,- auth,- )- .await?;+ crate::github_issues::fetch_all(http, &format!("{base}/issues"), &issue_q, auth).await?;let mut stats = SyncStats::default();- for (issue, cs) in crate::github_issues::group_comments(&issues, &comments)- .into_iter()- .filter(|(i, _)| include_prs || i.pull_request.is_none())+ for issue in issues+ .iter()+ .filter(|i| include_prs || i.pull_request.is_none()).take(MAX_NEW_PER_SYNC){+ // 只重建本轮选中的工单,但每张要读完整评论历史。仓库级评论即使+ // 去掉 since,也可能被无关工单和 PR 的评论挤出分页上限。+ let comments = crate::github_issues::fetch_all::<crate::github_issues::Comment>(+ http,+ &format!("{base}/issues/{}/comments", issue.number),+ &[],+ auth,+ )+ .await?;+ let cs = crate::github_issues::comments_for_issue(issue, &comments);// 逐工单取事件。N 只是本轮要写入的工单数——首次同步等于总数,// 之后有 since 兜着通常是个位数let events = crate::github_issues::sort_events(crate::github_issues::fetch_all(- &http,+ http,&format!("{base}/issues/{}/events", issue.number),&[],auth,@@ -1239,3 +1251,7 @@ mod tests {#[cfg(test)]#[path = "source_filename_tests.rs"]mod source_filename_tests;++#[cfg(test)]+#[path = "github_sync_tests.rs"]+mod github_sync_tests;