Skip to content

World's first open-source enterprise world model.

A base can follow a GitHub repository's markdown files, fetching only the files that changed since the last sync (#1325)

Quintin Botesd62e880b15ba
diff --git a/Cargo.lock b/Cargo.lock
index 123cae83..23de9c05 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5514,6 +5514,7 @@ dependencies = [
"feed-rs",
"flate2",
"futures-util",
+ "glob",
"jsonwebtoken",
"object_store",
"oxrdf",
@@ -5525,6 +5526,7 @@ dependencies = [
"rig-core",
"serde",
"serde_json",
+ "sha1",
"sha2 0.11.0",
"sqlparser",
"sqlx",
diff --git a/Cargo.toml b/Cargo.toml
index fc845702..fc843bd7 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -78,6 +78,8 @@ feed-rs = "2"
cron = "0.12"
infer = "0.22"
sha2 = "0.11"
+glob = "0.3"
+sha1 = "0.10"
dom_smoothie = "0.16"
dom_query = "0.26"
htmd = "0.5"
diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs
index 4bc79d37..a32be021 100644
--- a/crates/utopia-core/src/models.rs
+++ b/crates/utopia-core/src/models.rs
@@ -271,6 +271,7 @@ pub enum SourceKind {
Url,
Rss,
GithubIssues,
+ GithubRepo,
JiraIssues,
S3,
AzureBlob,
diff --git a/crates/utopia-server/Cargo.toml b/crates/utopia-server/Cargo.toml
index 6cf9f297..e7f24d44 100644
--- a/crates/utopia-server/Cargo.toml
+++ b/crates/utopia-server/Cargo.toml
@@ -17,6 +17,8 @@ utopia-search.workspace = true
utopia-llm.workspace = true
utopia-extract = { path = "../utopia-extract" }
sha2.workspace = true
+glob.workspace = true
+sha1.workspace = true
futures-util.workspace = true
async-stream.workspace = true
tokio.workspace = true
diff --git a/crates/utopia-server/src/github_issues.rs b/crates/utopia-server/src/github_issues.rs
index dffe3e66..0b725466 100644
--- a/crates/utopia-server/src/github_issues.rs
+++ b/crates/utopia-server/src/github_issues.rs
@@ -209,6 +209,41 @@ pub fn sort_events(mut events: Vec<Event>) -> Vec<Event> {
events
}
+/// 发一个带令牌的 GET 并判状态。github_issues 与 github_repo 共用这一条,
+/// 免得限流提示在两个来源里各写一份、慢慢漂移。
+pub(crate) async fn get_checked(
+ http: &reqwest::Client,
+ url: reqwest::Url,
+ auth: Option<&str>,
+ accept: Option<&str>,
+) -> anyhow::Result<reqwest::Response> {
+ let mut req = http.get(url);
+ if let Some(a) = auth {
+ req = req.header(reqwest::header::AUTHORIZATION, a);
+ }
+ if let Some(a) = accept {
+ req = req.header(reqwest::header::ACCEPT, a);
+ }
+ let resp = req.send().await?;
+ let status = resp.status();
+ if !status.is_success() {
+ // 限流说清楚是限流:未认证时每小时 60 次,一个中等仓库一次同步就能吃光。
+ // 报成通用 HTTP 错会让人去查网络,而正确的动作是配一个令牌
+ let remaining = resp
+ .headers()
+ .get("x-ratelimit-remaining")
+ .and_then(|v| v.to_str().ok())
+ .unwrap_or("?");
+ if status == reqwest::StatusCode::FORBIDDEN && remaining == "0" {
+ anyhow::bail!(
+ "GitHub 限流:本小时配额已用尽。未认证时每小时 60 次,配一个令牌可提到 5000"
+ );
+ }
+ anyhow::bail!("HTTP {status} from GitHub");
+ }
+ Ok(resp)
+}
+
/// 分页取一个端点,直到空页或触顶。
pub async fn fetch_all<T: for<'de> Deserialize<'de>>(
http: &reqwest::Client,
@@ -229,27 +264,7 @@ pub async fn fetch_all<T: for<'de> Deserialize<'de>>(
q.append_pair("per_page", &PER_PAGE.to_string());
q.append_pair("page", &page.to_string());
}
- let mut req = http.get(url);
- if let Some(a) = auth {
- req = req.header(reqwest::header::AUTHORIZATION, a);
- }
- let resp = req.send().await?;
- let status = resp.status();
- if !status.is_success() {
- // 限流说清楚是限流:未认证时每小时 60 次,一个中等仓库一次同步就能吃光。
- // 报成通用 HTTP 错会让人去查网络,而正确的动作是配一个令牌
- let remaining = resp
- .headers()
- .get("x-ratelimit-remaining")
- .and_then(|v| v.to_str().ok())
- .unwrap_or("?");
- if status == reqwest::StatusCode::FORBIDDEN && remaining == "0" {
- anyhow::bail!(
- "GitHub 限流:本小时配额已用尽。未认证时每小时 60 次,配一个令牌可提到 5000"
- );
- }
- anyhow::bail!("HTTP {status} from GitHub");
- }
+ let resp = get_checked(http, url, auth, None).await?;
let batch: Vec<T> = resp.json().await?;
let n = batch.len();
out.extend(batch);
diff --git a/crates/utopia-server/src/github_repo.rs b/crates/utopia-server/src/github_repo.rs
new file mode 100644
index 00000000..b3d7d5c4
--- /dev/null
+++ b/crates/utopia-server/src/github_repo.rs
@@ -0,0 +1,117 @@
+//! GitHub 仓库文件来源:默认分支头的一棵文件树,一条路径一篇文档。
+//! 同步在 `ingest_sources`;这里只放不联网的那一半,于是单测得动。
+
+use serde::Deserialize;
+use sha1::Digest as _;
+use std::collections::HashSet;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct TreeEntry {
+ pub path: String,
+ pub mode: String,
+ #[serde(rename = "type")]
+ pub kind: String,
+ pub sha: String,
+ pub size: Option<i64>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TreeResponse {
+ pub tree: Vec<TreeEntry>,
+ #[serde(default)]
+ pub truncated: bool,
+}
+
+/// 哪条路径收、收多大的文件。只收 markdown 是为了不把整个代码仓当语料倒进来。
+pub struct FileFilter {
+ include: Vec<glob::Pattern>,
+ exclude: Vec<glob::Pattern>,
+ max_file_bytes: i64,
+}
+
+impl FileFilter {
+ pub fn from_config(config: &serde_json::Value) -> anyhow::Result<Self> {
+ // 配置给了这一列就整列替换默认值
+ let compile = |key: &str, defaults: &[&str]| -> anyhow::Result<Vec<glob::Pattern>> {
+ let list: Vec<&str> = match config[key].as_array() {
+ Some(a) => a.iter().filter_map(|v| v.as_str()).collect(),
+ None => defaults.to_vec(),
+ };
+ list.into_iter()
+ .map(|p| {
+ glob::Pattern::new(p)
+ .map_err(|e| anyhow::anyhow!("config.{key} has an invalid glob {p:?}: {e}"))
+ })
+ .collect()
+ };
+ Ok(Self {
+ include: compile("include", &["**/*.md", "**/*.markdown", "**/*.mdx"])?,
+ exclude: compile(
+ "exclude",
+ &["**/.*/**", "**/node_modules/**", "**/vendor/**"],
+ )?,
+ max_file_bytes: config["max_file_bytes"].as_i64().unwrap_or(1 << 20),
+ })
+ }
+
+ fn eligible(&self, e: &TreeEntry) -> bool {
+ let opts = glob::MatchOptions {
+ case_sensitive: true,
+ require_literal_separator: true,
+ require_literal_leading_dot: false,
+ };
+ // 子模块是 commit、目录是 tree,符号链接指向别处——都不是文件本体
+ e.kind == "blob"
+ && e.mode != "120000"
+ && e.size.is_some_and(|s| s <= self.max_file_bytes)
+ && self.include.iter().any(|p| p.matches_with(&e.path, opts))
+ && !self.exclude.iter().any(|p| p.matches_with(&e.path, opts))
+ }
+}
+
+/// 按路径排序、同一 blob 只留第一条:相同文件在几个路径出现时,文档不该在路径之间横跳。
+pub fn choose(entries: Vec<TreeEntry>, filter: &FileFilter) -> Vec<TreeEntry> {
+ let mut entries: Vec<_> = entries.into_iter().filter(|e| filter.eligible(e)).collect();
+ entries.sort_by(|a, b| a.path.cmp(&b.path));
+ let mut seen = HashSet::new();
+ entries.retain(|e| seen.insert(e.sha.clone()));
+ entries
+}
+
+/// git 里一个 blob 的 SHA-1(先喂 `blob <字节数>\0` 再喂正文)。树里给的就是这个值,
+/// 拿已存的字节算一遍就知道没变,不必再下文件。
+pub fn git_blob_sha(bytes: &[u8]) -> String {
+ let mut h = sha1::Sha1::new();
+ h.update(format!("blob {}\0", bytes.len()).as_bytes());
+ h.update(bytes);
+ h.finalize().iter().map(|b| format!("{b:02x}")).collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn the_default_globs_take_markdown_outside_hidden_and_vendored_folders() {
+ let f = FileFilter::from_config(&serde_json::json!({})).unwrap();
+ let entry = |path: &str, mode: &str| TreeEntry {
+ path: path.into(),
+ mode: mode.into(),
+ kind: "blob".into(),
+ sha: "s".into(),
+ size: Some(10),
+ };
+ for path in ["README.md", "docs/a/b.mdx"] {
+ assert!(f.eligible(&entry(path, "100644")), "{path}");
+ }
+ for path in [
+ ".github/PULL_REQUEST_TEMPLATE.md",
+ "web/node_modules/x/README.md",
+ "vendor/a.md",
+ "Cargo.toml",
+ ] {
+ assert!(!f.eligible(&entry(path, "100644")), "{path}");
+ }
+ assert!(!f.eligible(&entry("link.md", "120000")));
+ }
+}
diff --git a/crates/utopia-server/src/github_repo_sync_tests.rs b/crates/utopia-server/src/github_repo_sync_tests.rs
new file mode 100644
index 00000000..9ce35c92
--- /dev/null
+++ b/crates/utopia-server/src/github_repo_sync_tests.rs
@@ -0,0 +1,147 @@
+//! 用录下的真实 GitHub 响应跑仓库同步:筛选、同一 blob 只收一处、没变不再取、
+//! 树截断时不判「不在来源中」。
+use super::sync_github_repo_with_client;
+use serde_json::{json, Value};
+use std::sync::{Arc, Mutex};
+use uuid::Uuid;
+use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate};
+
+const PACKS: &str = "crates/utopia-server/packs/README.md";
+const TESTDATA: &str = "crates/utopia-server/testdata/README.md";
+
+/// 录下的树再加一个 testdata/README.md 的副本。阶段 1、2 去掉 packs,阶段 1 还标了截断
+fn tree(fixture: &Value, stage: u8) -> Value {
+ let mut entries = fixture["tree"]["tree"].as_array().unwrap().clone();
+ let mut copy = entries
+ .iter()
+ .find(|e| e["path"] == TESTDATA)
+ .unwrap()
+ .clone();
+ copy["path"] = json!("docs/copy/README.md");
+ entries.push(copy);
+ if stage > 0 {
+ entries.retain(|e| e["path"] != PACKS);
+ }
+ json!({ "tree": entries, "truncated": stage == 1 })
+}
+
+#[tokio::test]
+async fn a_repository_sync_takes_each_changed_file_once() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = sqlx::PgPool::connect(&url).await?;
+ let (org, ws, kb, sid) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ let result = async {
+ // 2000 字节的上限把 2920 字节的 SECURITY.md 挡在外面
+ sqlx::raw_sql(&format!(
+ r#"INSERT INTO organizations(id,name) VALUES('{org}','repo-test');
+ INSERT INTO workspaces(id,org_id,name) VALUES('{ws}','{org}','repo-test');
+ INSERT INTO knowledge_bases(id,workspace_id,name) VALUES('{kb}','{ws}','repo-test');
+ INSERT INTO sources(id,kb_id,kind,name,config)
+ VALUES('{sid}','{kb}','github_repo','fixture',
+ '{{"repo":"deeplethe/utopia","max_file_bytes":2000}}');"#
+ ))
+ .execute(&pool)
+ .await?;
+ let source = utopia_store::sources::get(&pool, sid).await?;
+ let dir = tempfile::tempdir()?;
+ let cfg = utopia_core::config::AppConfig {
+ data_dir: dir.path().to_string_lossy().into_owned(),
+ ..Default::default()
+ };
+ let search = Arc::new(utopia_search::SearchIndex::open(
+ &dir.path().join("search"),
+ )?);
+ let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into());
+
+ let fixture: Value =
+ serde_json::from_str(include_str!("../tests/fixtures/github_repo.json"))?;
+ let server = MockServer::start().await;
+ let stage = Arc::new(Mutex::new(0u8));
+ let (current, fx) = (stage.clone(), fixture.clone());
+ Mock::given(method("GET"))
+ .respond_with(move |request: &Request| {
+ let path = request.url.path();
+ let body = if path.ends_with("/git/trees/HEAD") {
+ tree(&fx, *current.lock().unwrap())
+ } else if path.ends_with("/commits") {
+ let (_, of) = request
+ .url
+ .query_pairs()
+ .find(|(k, _)| k == "path")
+ .unwrap();
+ fx["commits"][&*of].clone()
+ } else if let Some((_, sha)) = path.split_once("/git/blobs/") {
+ return ResponseTemplate::new(200)
+ .set_body_string(fx["blobs"][sha].as_str().unwrap());
+ } else {
+ return ResponseTemplate::new(404);
+ };
+ ResponseTemplate::new(200).set_body_json(body)
+ })
+ .mount(&server)
+ .await;
+ let (http, uri) = (reqwest::Client::builder().no_proxy().build()?, server.uri());
+ let sync = || sync_github_repo_with_client(&state, &source, &http, &uri);
+ let doc = |path: &str| {
+ let key = format!("github:deeplethe/utopia:{path}");
+ let pool = pool.clone();
+ async move { utopia_store::documents::find_by_external_key(&pool, sid, &key).await }
+ };
+
+ let first = sync().await?;
+ for path in [PACKS, TESTDATA] {
+ let date = fixture["commits"][path][0]["commit"]["committer"]["date"].as_str();
+ let time = doc(path).await?.and_then(|d| d.doc_time);
+ anyhow::ensure!(
+ time == date.and_then(|d| d.parse().ok()),
+ "{path}:{time:?}"
+ );
+ }
+ for path in [
+ "SECURITY.md",
+ ".github/PULL_REQUEST_TEMPLATE.md",
+ "docs/copy/README.md",
+ ] {
+ anyhow::ensure!(doc(path).await?.is_none(), "{path} 不该收");
+ }
+ anyhow::ensure!(first.created == 2, "第一轮建了 {}", first.created);
+
+ let seen = server.received_requests().await.unwrap().len();
+ let second = sync().await?;
+ let refetched = server.received_requests().await.unwrap()[seen..]
+ .iter()
+ .filter(|r| !r.url.path().ends_with("/git/trees/HEAD"))
+ .count();
+ anyhow::ensure!(
+ second.total() == 0 && refetched == 0,
+ "没变又取了 {refetched} 次"
+ );
+
+ *stage.lock().unwrap() = 1;
+ sync().await?;
+ let missing = || async { anyhow::Ok(doc(PACKS).await?.unwrap().missing_since) };
+ anyhow::ensure!(missing().await?.is_none(), "截断的树不该判消失");
+ *stage.lock().unwrap() = 2;
+ sync().await?;
+ anyhow::ensure!(missing().await?.is_some(), "完整的树里没了才判消失");
+ anyhow::ensure!(doc(TESTDATA).await?.unwrap().missing_since.is_none());
+ anyhow::Ok(())
+ }
+ .await;
+ sqlx::query("DELETE FROM knowledge_bases WHERE id=$1")
+ .bind(kb)
+ .execute(&pool)
+ .await?;
+ sqlx::query("DELETE FROM organizations WHERE id=$1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ result
+}
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs
index 17469be0..26e64209 100644
--- a/crates/utopia-server/src/ingest_sources.rs
+++ b/crates/utopia-server/src/ingest_sources.rs
@@ -83,6 +83,7 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<()
Some(SourceKind::Rss) => sync_rss(state, &source).await,
Some(SourceKind::Custom) => sync_custom(state, &source, since).await,
Some(SourceKind::GithubIssues) => sync_github_issues(state, &source, since).await,
+ Some(SourceKind::GithubRepo) => sync_github_repo(state, &source).await,
Some(SourceKind::JiraIssues) => sync_jira_issues(state, &source, since).await,
Some(SourceKind::S3 | SourceKind::AzureBlob | SourceKind::Gcs) => {
sync_object_storage(state, &source).await
@@ -886,6 +887,106 @@ async fn sync_github_issues_with_client(
Ok(stats)
}
+/// GitHub 仓库文件:一次递归取默认分支的整棵树,一条路径一篇文档。
+/// 树里的 blob SHA 就判得出「没变」,变了的才逐条取提交日期与原文。
+async fn sync_github_repo(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {
+ // 与 github_issues 同一个口径:固定的公网主机按内容级严格度收
+ let url = reqwest::Url::parse("https://api.github.com/")?;
+ let http = crate::http_fetch::client_for(
+ &url,
+ crate::http_fetch::Reach::Content,
+ crate::http_fetch::Limits::default(),
+ )
+ .await?;
+ sync_github_repo_with_client(state, source, &http, "https://api.github.com").await
+}
+
+// HTTP 接缝让回归测试走同一条同步与落库路径,生产仍固定访问 GitHub 公网 API。
+async fn sync_github_repo_with_client(
+ state: &AppState,
+ source: &Source,
+ http: &reqwest::Client,
+ api_base: &str,
+) -> anyhow::Result<SyncStats> {
+ use crate::{github_issues::get_checked, github_repo as gr};
+ let repo = source.config["repo"]
+ .as_str()
+ .map(str::trim)
+ .filter(|s| !s.is_empty())
+ .ok_or_else(|| anyhow::anyhow!("github_repo source is missing config.repo (owner/name)"))?;
+ if repo.split('/').count() != 2 {
+ anyhow::bail!("config.repo should look like owner/name, got {repo:?}");
+ }
+ let auth = source.config["auth_header"]
+ .as_str()
+ .map(str::trim)
+ .filter(|s| !s.is_empty());
+ let filter = gr::FileFilter::from_config(&source.config)?;
+ let base = format!("{api_base}/repos/{repo}");
+ let tree_url = reqwest::Url::parse(&format!("{base}/git/trees/HEAD?recursive=1"))?;
+ let tree: gr::TreeResponse = get_checked(http, tree_url, auth, None)
+ .await?
+ .json()
+ .await?;
+ let (truncated, chosen) = (tree.truncated, gr::choose(tree.tree, &filter));
+ let keys: Vec<String> = chosen
+ .iter()
+ .map(|e| format!("github:{repo}:{}", e.path))
+ .collect();
+ let mut stats = SyncStats::default();
+ for (entry, key) in chosen.iter().zip(&keys) {
+ // 没变的整条跳过:已落库的字节算出同一个 git SHA,就不必再花两次请求
+ let old =
+ utopia_store::documents::find_by_external_key(&state.pool, source.id, key).await?;
+ let unchanged = match old {
+ Some(d) if Some(d.size_bytes) == entry.size => state
+ .blob
+ .get(&d.sha256)
+ .await
+ .is_ok_and(|b| gr::git_blob_sha(&b) == entry.sha),
+ _ => false,
+ };
+ if unchanged {
+ continue;
+ }
+ // 文档日期取最后一次动过这条路径的提交,只在新出现或变了时问
+ let mut cu = reqwest::Url::parse(&format!("{base}/commits"))?;
+ cu.query_pairs_mut()
+ .append_pair("path", &entry.path)
+ .append_pair("per_page", "1");
+ let commits: serde_json::Value = get_checked(http, cu, auth, None).await?.json().await?;
+ let doc_time = commits[0]["commit"]["committer"]["date"]
+ .as_str()
+ .and_then(|d| d.parse().ok());
+ let bu = reqwest::Url::parse(&format!("{base}/git/blobs/{}", entry.sha))?;
+ let bytes = get_checked(http, bu, auth, Some("application/vnd.github.raw+json"))
+ .await?
+ .bytes()
+ .await?;
+ let action = ingest_item(
+ state,
+ source.kb_id,
+ source.id,
+ key,
+ &entry.path,
+ "text/markdown",
+ &bytes,
+ doc_time,
+ )
+ .await?;
+ stats.absorb(action);
+ }
+ if truncated {
+ // 树不全就不判「谁消失了」——缺席可能只是还没列到
+ tracing::warn!(%repo, "GitHub 文件树被截断,本轮只摄入列出的文件,不做不在来源中对账");
+ } else {
+ utopia_store::documents::reconcile_missing(&state.pool, source.id, &keys)
+ .await
+ .map_err(|e| anyhow::anyhow!(e))?;
+ }
+ Ok(stats)
+}
+
/// Jira 工单:一张工单一篇文档,正文里带**字段级**的变更史。
///
/// 比 GitHub 那条路省:`search?expand=changelog` 一次调用就带回工单本体、
@@ -1311,3 +1412,7 @@ mod source_filename_tests;
#[cfg(test)]
#[path = "github_sync_tests.rs"]
mod github_sync_tests;
+
+#[cfg(test)]
+#[path = "github_repo_sync_tests.rs"]
+mod github_repo_sync_tests;
diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs
index 6617ea07..3ad724e3 100644
--- a/crates/utopia-server/src/main.rs
+++ b/crates/utopia-server/src/main.rs
@@ -10,6 +10,7 @@ mod error;
mod extraction;
mod extraction_open;
mod github_issues;
+mod github_repo;
mod governance;
mod http_fetch;
mod implication;
diff --git a/crates/utopia-server/tests/fixtures/github_repo.json b/crates/utopia-server/tests/fixtures/github_repo.json
new file mode 100644
index 00000000..c420115e
--- /dev/null
+++ b/crates/utopia-server/tests/fixtures/github_repo.json
@@ -0,0 +1,19 @@
+{
+ "note": "取自 deeplethe/utopia 的真实 GitHub 响应(树 cda3747),条目和字段已裁到我们读的那些。blobs 是 Accept: application/vnd.github.raw+json 取回的原文;commits 是 commits?path=…&per_page=1 的第一条。",
+ "tree": {"sha": "cda374771577b2d42ed257312864ecf8e067bfbd", "truncated": false, "tree": [
+ {"path": ".github/PULL_REQUEST_TEMPLATE.md", "mode": "100644", "type": "blob", "sha": "9b05a587001713b657e8cc51bb94f3a14add1dee", "size": 1067},
+ {"path": "Cargo.toml", "mode": "100644", "type": "blob", "sha": "5b65de568922150141e12599d393e21db58eb6e3", "size": 4030},
+ {"path": "SECURITY.md", "mode": "100644", "type": "blob", "sha": "274cff38310d7322ef84f27174e0344435ef5afc", "size": 2920},
+ {"path": "crates/utopia-server/packs/README.md", "mode": "100644", "type": "blob", "sha": "69dbb3020bce31ed843185f1dd24af44184a5781", "size": 1364},
+ {"path": "crates/utopia-server/testdata/README.md", "mode": "100644", "type": "blob", "sha": "2460624379264fed241f0986cd7dda7a5d86c104", "size": 1026},
+ {"path": "docs", "mode": "040000", "type": "tree", "sha": "c2e443619c8b74e52530a296f1282b77c3e4f954"}
+ ]},
+ "commits": {
+ "crates/utopia-server/packs/README.md": [{"sha": "92db693c3e4763f03551fcb1144684336ccd83f5", "commit": {"committer": {"date": "2026-08-30T01:17:00Z"}}}],
+ "crates/utopia-server/testdata/README.md": [{"sha": "798fb2476f4ec8eacf3d6220657f27333ad50d94", "commit": {"committer": {"date": "2026-09-30T10:04:58Z"}}}]
+ },
+ "blobs": {
+ "69dbb3020bce31ed843185f1dd24af44184a5781": "# 预制本体包\n\n来源文件按原样保存(gzip 压缩),由 `src/ontology_packs.rs` 用 `include_bytes!` 内嵌进二进制。\n\n**为什么内嵌而不是运行时下载**:README 承诺整套系统可以运行在完全离线的内网环境。\n运行时抓取会让这句话失效,也让构建不可复现(上游随时会发新版)。\n\n**为什么压缩**:未压缩合计 1.7 MB,压缩后 316 KB。这是个公开仓库,clone 体积是真实成本。\n`flate2` 本来就在依赖树里,解压是三行。\n\n| 文件 | 来源 | 许可 | 抓取日 |\n|---|---|---|---|\n| `schema-org.ttl.gz` | https://schema.org/version/latest/schemaorg-current-https.ttl | CC BY-SA 3.0 | 2026-08-30 |\n| `w3c-org.ttl.gz` | https://www.w3.org/ns/org.ttl | W3C Document License | 2026-08-30 |\n| `prov-o.ttl.gz` | https://www.w3.org/ns/prov.ttl | W3C Document License | 2026-08-30 |\n| `foaf.rdf.gz` | http://xmlns.com/foaf/spec/index.rdf | CC BY 1.0 | 2026-08-30 |\n| `iof-core.rdf.gz` | https://spec.industrialontologies.org/ontology/core/Core/ | MIT | 2026-08-30 |\n\n## 更新一个包\n\n重新抓取、`gzip -9c` 覆盖、更新 `ontology_packs.rs` 里的计数与本文件的抓取日。\n**不要改动原文内容**——投影只覆盖当下能消费的部分,原文保真是 [0001](../../../docs/decisions/0001-ontology-import-and-governance.md) 判据 1。\n",
+ "2460624379264fed241f0986cd7dda7a5d86c104": "# OCR page fixture\n\n`ark-ocr-pages.pdf` is a synthetic three-page PDF created for OCR acceptance tests.\nPages 1 and 3 contain rasterized Chinese and English text, dates, amounts, a small\ntable, an instruction printed as source text, and an unlabelled line chart. Page 2\nis blank. There is no text layer. All content is authored for this test and carries\nno third-party or private data.\n\nThe fixture tests real Poppler rendering, original page numbering after a blank\npage, and restarting a failed pass from page 1. Mock replies test those contracts;\nthey do not measure a model's recognition accuracy.\n\nWith `ARK_API_KEY` set, run the manual Agent Plan acceptance test:\n\n```sh\ncargo test -p utopia-server live_agent_plan_reads_scans -- --ignored --nocapture\n```\n\nThis sends the probe and this fixture to the real API. Optional\n`ARK_OCR_TEST_PDF` adds a public scan, and `ARK_OCR_TEST_OUTPUT` saves its text and\nprovenance without credentials. `ARK_OCR_TEST_BASE_URL` and\n`ARK_OCR_TEST_MODEL` override the Agent Plan defaults.\n"
+ }
+}
diff --git a/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs b/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs
index 269f9feb..68674ec1 100644
--- a/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs
+++ b/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs
@@ -56,6 +56,10 @@ async fn a_viewer_never_sees_a_credential() -> anyhow::Result<()> {
"github_issues",
serde_json::json!({ "repo": "o/r", "auth_header": "Bearer g" }),
),
+ (
+ "github_repo",
+ serde_json::json!({ "repo": "o/r", "auth_header": "Bearer r" }),
+ ),
(
"jira_issues",
serde_json::json!({ "base_url": "https://j.test", "project": "P", "auth_header": "Basic j" }),
diff --git a/docs/design/sources.md b/docs/design/sources.md
index bbc34da1..d543cec1 100644
--- a/docs/design/sources.md
+++ b/docs/design/sources.md
@@ -19,7 +19,9 @@ document date, so an upload-dated document counts as undated for attestation and
web list derive from; credentials never leave the server (`SOURCE_SECRET_KEYS`) and a blank field
keeps the stored value; a truncated round says so [0013]. A ticket is one document whose body
carries its history as dated declarative sentences and a `## History` list; GitHub fetches events
-per ticket; timestamps are written to the second [0013].
+per ticket; timestamps are written to the second [0013]. A GitHub repository file is one document
+keyed by its path and dated by the last commit that touched it; a file whose blob SHA has not
+changed is not fetched again, and identical files at several paths go in once [#1303].
**Statements.** A `statements` source takes the open extraction contract itself (`e`, `s`, `n`) on `POST /sources/{id}/statements`, with the `api` push's identity, versions, tombstones and run history; the payload is the document and its single chunk, and extraction parses the chunk instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement with the item as its evidence (null offsets) and reaches the typed graph only through alignment. The contract has no slot for a property or a class and unknown keys are refused; a second push under the same identity marks the earlier statements stale rather than closing them; tables belong on a mount, not here [0054].
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index a7109fcb..71284011 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -640,6 +640,7 @@ export const en = {
statements: "Statements",
custom: "Custom",
github_issues: "GitHub issues",
+ github_repo: "GitHub repository files",
jira_issues: "Jira issues",
s3: "S3 / MinIO",
azure_blob: "Azure Blob",
@@ -659,6 +660,9 @@ export const en = {
"Syncs a repository's issues. **Each ticket, together with its state history**, becomes " +
"one document — when it was opened, closed, relabelled, reassigned, all dated. " +
"Without a token GitHub allows only 60 requests an hour.",
+ github_repo:
+ "Reads markdown files off a repository's default branch. Each file becomes one " +
+ "document dated by its last commit; unchanged files are not fetched again.",
s3:
"Reads documents out of an S3 bucket, or anything speaking the same protocol " +
"(MinIO, Ceph, R2). **Leave the endpoint empty for AWS**; fill it in for a " +
@@ -756,6 +760,8 @@ export const en = {
notionQueryField: "Search term (optional — empty takes every shared page)",
tokenField: "GitHub token (optional, never shown again)",
includePullRequests: "Treat pull requests as tickets too",
+ includeGlobsField: "Include globs (comma-separated, optional)",
+ excludeGlobsField: "Exclude globs (comma-separated, optional)",
interval: "Sync schedule",
intervalManual: "Manual only",
intervalEvery: (m: number) =>
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 9222d4ed..e7485665 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -579,6 +579,7 @@ export const zh: Strings = {
statements: "陈述",
custom: "自定义",
github_issues: "GitHub 工单",
+ github_repo: "GitHub 仓库文件",
jira_issues: "Jira 工单",
s3: "S3 / MinIO",
azure_blob: "Azure Blob",
@@ -597,6 +598,9 @@ export const zh: Strings = {
"同步一个仓库的工单。**每张工单连同它的状态变更史**成为一篇文档——" +
"何时开出、何时关闭、标签与负责人怎么变的,都带着日期。未配令牌时" +
"每小时只有 60 次请求配额。",
+ github_repo:
+ "读取一个仓库默认分支上的 markdown 文件。每个文件成为一篇文档," +
+ "日期取最后一次改动它的提交;没变的文件不会重抓。",
s3:
"从 S3 桶里读文档,或者任何说同一套协议的东西(MinIO、Ceph、R2)。" +
"**端点留空就是 AWS**,填了就是自建。每个对象成为一篇文档," +
@@ -689,6 +693,8 @@ export const zh: Strings = {
notionQueryField: "搜索词(可选——留空则取全部已分享页面)",
tokenField: "GitHub 令牌(可选,不再显示)",
includePullRequests: "把 PR 也当工单收进来",
+ includeGlobsField: "包含的 glob(逗号分隔,可选)",
+ excludeGlobsField: "排除的 glob(逗号分隔,可选)",
interval: "同步计划",
intervalManual: "仅手动",
intervalEvery: (m: number) =>
diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsx
index 3fe65b45..40e314a0 100644
--- a/web/src/pages/Library.tsx
+++ b/web/src/pages/Library.tsx
@@ -1360,6 +1360,8 @@ function SourceModal({
// PR 在 GitHub 的模型里也是工单。默认不收——问「工单」要的是工单;
// 但有些仓库的决策记录实际写在 PR 描述里,所以给个开关
const [includePrs, setIncludePrs] = useState(false);
+ const [includeGlobs, setIncludeGlobs] = useState("");
+ const [excludeGlobs, setExcludeGlobs] = useState("");
const [schedule, setSchedule] = useState<ScheduleValue>({
sync_interval_minutes: null,
sync_cron: null,
@@ -1370,6 +1372,7 @@ function SourceModal({
kind === "rss" ||
kind === "custom" ||
kind === "github_issues" ||
+ kind === "github_repo" ||
kind === "jira_issues" ||
kind === "s3" ||
kind === "azure_blob" ||
@@ -1379,6 +1382,11 @@ function SourceModal({
const create = useMutation({
mutationFn: () => {
+ // 留空就不发,后端按默认的 markdown 规则筛
+ const globs = (key: string, value: string) => {
+ const list = value.split(",").map((g) => g.trim()).filter(Boolean);
+ return list.length ? { [key]: list } : {};
+ };
const config =
kind === "url"
? { urls: urls.split("\n").map((u) => u.trim()).filter(Boolean) }
@@ -1389,11 +1397,13 @@ function SourceModal({
endpoint: endpoint.trim(),
...(authHeader.trim() ? { auth_header: authHeader.trim() } : {}),
}
- : kind === "github_issues"
+ : kind === "github_issues" || kind === "github_repo"
? {
repo: repo.trim(),
...(authHeader.trim() ? { auth_header: authHeader.trim() } : {}),
- ...(includePrs ? { include_pull_requests: true } : {}),
+ ...(kind === "github_issues"
+ ? includePrs ? { include_pull_requests: true } : {}
+ : { ...globs("include", includeGlobs), ...globs("exclude", excludeGlobs) }),
}
: kind === "jira_issues"
? {
@@ -1799,7 +1809,7 @@ function SourceModal({
)}
</>
)}
- {kind === "github_issues" && (
+ {(kind === "github_issues" || kind === "github_repo") && (
<>
{field(
S.library.repoField,
@@ -1818,12 +1828,33 @@ function SourceModal({
onChange={(e) => setAuthHeader(e.target.value)}
/>,
)}
- <Checkbox
- className="mb-4"
- checked={includePrs}
- onChange={(v) => setIncludePrs(v)}
- label={S.library.includePullRequests}
- />
+ {kind === "github_issues" ? (
+ <Checkbox
+ className="mb-4"
+ checked={includePrs}
+ onChange={(v) => setIncludePrs(v)}
+ label={S.library.includePullRequests}
+ />
+ ) : (
+ <>
+ {field(
+ S.library.includeGlobsField,
+ <Input className="w-full font-mono"
+ placeholder="**/*.md, **/*.markdown, **/*.mdx"
+ value={includeGlobs}
+ onChange={(e) => setIncludeGlobs(e.target.value)}
+ />,
+ )}
+ {field(
+ S.library.excludeGlobsField,
+ <Input className="w-full font-mono"
+ placeholder="**/.*/**, **/node_modules/**, **/vendor/**"
+ value={excludeGlobs}
+ onChange={(e) => setExcludeGlobs(e.target.value)}
+ />,
+ )}
+ </>
+ )}
</>
)}
diff --git a/web/src/pages/SourcesRail.tsx b/web/src/pages/SourcesRail.tsx
index 9929bb0b..e881dfba 100644
--- a/web/src/pages/SourcesRail.tsx
+++ b/web/src/pages/SourcesRail.tsx
@@ -72,6 +72,7 @@ export const KIND_ICON: Record<SourceKind, LucideIcon> = {
statements: Braces,
custom: Puzzle,
github_issues: CircleDot,
+ github_repo: FileText,
jira_issues: SquareKanban,
s3: HardDrive,
azure_blob: Cloud,
@@ -88,6 +89,7 @@ export const SYNCING_KINDS = new Set([
"rss",
"custom",
"github_issues",
+ "github_repo",
"jira_issues",
"s3",
"azure_blob",
diff --git a/web/src/sourceKinds.ts b/web/src/sourceKinds.ts
index c6e30af4..35c82f18 100644
--- a/web/src/sourceKinds.ts
+++ b/web/src/sourceKinds.ts
@@ -13,6 +13,7 @@ export const CREATABLE_SOURCE_KINDS = [
"url",
"rss",
"github_issues",
+ "github_repo",
"jira_issues",
"s3",
"azure_blob",