World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A base can follow a GitHub repository's markdown files, fetching only the files that changed since the last sync (#1325)
Quintin Botesd62e880b15ba
diff --git a/Cargo.lock b/Cargo.lockindex 123cae83..23de9c05 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5514,6 +5514,7 @@ dependencies = ["feed-rs","flate2","futures-util",+ "glob","jsonwebtoken","object_store","oxrdf",@@ -5525,6 +5526,7 @@ dependencies = ["rig-core","serde","serde_json",+ "sha1","sha2 0.11.0","sqlparser","sqlx",diff --git a/Cargo.toml b/Cargo.tomlindex fc845702..fc843bd7 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -78,6 +78,8 @@ feed-rs = "2"cron = "0.12"infer = "0.22"sha2 = "0.11"+glob = "0.3"+sha1 = "0.10"dom_smoothie = "0.16"dom_query = "0.26"htmd = "0.5"diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rsindex 4bc79d37..a32be021 100644--- a/crates/utopia-core/src/models.rs+++ b/crates/utopia-core/src/models.rs@@ -271,6 +271,7 @@ pub enum SourceKind {Url,Rss,GithubIssues,+ GithubRepo,JiraIssues,S3,AzureBlob,diff --git a/crates/utopia-server/Cargo.toml b/crates/utopia-server/Cargo.tomlindex 6cf9f297..e7f24d44 100644--- a/crates/utopia-server/Cargo.toml+++ b/crates/utopia-server/Cargo.toml@@ -17,6 +17,8 @@ utopia-search.workspace = trueutopia-llm.workspace = trueutopia-extract = { path = "../utopia-extract" }sha2.workspace = true+glob.workspace = true+sha1.workspace = truefutures-util.workspace = trueasync-stream.workspace = truetokio.workspace = truediff --git a/crates/utopia-server/src/github_issues.rs b/crates/utopia-server/src/github_issues.rsindex dffe3e66..0b725466 100644--- a/crates/utopia-server/src/github_issues.rs+++ b/crates/utopia-server/src/github_issues.rs@@ -209,6 +209,41 @@ pub fn sort_events(mut events: Vec<Event>) -> Vec<Event> {events}+/// 发一个带令牌的 GET 并判状态。github_issues 与 github_repo 共用这一条,+/// 免得限流提示在两个来源里各写一份、慢慢漂移。+pub(crate) async fn get_checked(+ http: &reqwest::Client,+ url: reqwest::Url,+ auth: Option<&str>,+ accept: Option<&str>,+) -> anyhow::Result<reqwest::Response> {+ let mut req = http.get(url);+ if let Some(a) = auth {+ req = req.header(reqwest::header::AUTHORIZATION, a);+ }+ if let Some(a) = accept {+ req = req.header(reqwest::header::ACCEPT, a);+ }+ let resp = req.send().await?;+ let status = resp.status();+ if !status.is_success() {+ // 限流说清楚是限流:未认证时每小时 60 次,一个中等仓库一次同步就能吃光。+ // 报成通用 HTTP 错会让人去查网络,而正确的动作是配一个令牌+ let remaining = resp+ .headers()+ .get("x-ratelimit-remaining")+ .and_then(|v| v.to_str().ok())+ .unwrap_or("?");+ if status == reqwest::StatusCode::FORBIDDEN && remaining == "0" {+ anyhow::bail!(+ "GitHub 限流:本小时配额已用尽。未认证时每小时 60 次,配一个令牌可提到 5000"+ );+ }+ anyhow::bail!("HTTP {status} from GitHub");+ }+ Ok(resp)+}+/// 分页取一个端点,直到空页或触顶。pub async fn fetch_all<T: for<'de> Deserialize<'de>>(http: &reqwest::Client,@@ -229,27 +264,7 @@ pub async fn fetch_all<T: for<'de> Deserialize<'de>>(q.append_pair("per_page", &PER_PAGE.to_string());q.append_pair("page", &page.to_string());}- let mut req = http.get(url);- if let Some(a) = auth {- req = req.header(reqwest::header::AUTHORIZATION, a);- }- let resp = req.send().await?;- let status = resp.status();- if !status.is_success() {- // 限流说清楚是限流:未认证时每小时 60 次,一个中等仓库一次同步就能吃光。- // 报成通用 HTTP 错会让人去查网络,而正确的动作是配一个令牌- let remaining = resp- .headers()- .get("x-ratelimit-remaining")- .and_then(|v| v.to_str().ok())- .unwrap_or("?");- if status == reqwest::StatusCode::FORBIDDEN && remaining == "0" {- anyhow::bail!(- "GitHub 限流:本小时配额已用尽。未认证时每小时 60 次,配一个令牌可提到 5000"- );- }- anyhow::bail!("HTTP {status} from GitHub");- }+ let resp = get_checked(http, url, auth, None).await?;let batch: Vec<T> = resp.json().await?;let n = batch.len();out.extend(batch);diff --git a/crates/utopia-server/src/github_repo.rs b/crates/utopia-server/src/github_repo.rsnew file mode 100644index 00000000..b3d7d5c4--- /dev/null+++ b/crates/utopia-server/src/github_repo.rs@@ -0,0 +1,117 @@+//! GitHub 仓库文件来源:默认分支头的一棵文件树,一条路径一篇文档。+//! 同步在 `ingest_sources`;这里只放不联网的那一半,于是单测得动。++use serde::Deserialize;+use sha1::Digest as _;+use std::collections::HashSet;++#[derive(Debug, Clone, Deserialize)]+pub struct TreeEntry {+ pub path: String,+ pub mode: String,+ #[serde(rename = "type")]+ pub kind: String,+ pub sha: String,+ pub size: Option<i64>,+}++#[derive(Debug, Deserialize)]+pub struct TreeResponse {+ pub tree: Vec<TreeEntry>,+ #[serde(default)]+ pub truncated: bool,+}++/// 哪条路径收、收多大的文件。只收 markdown 是为了不把整个代码仓当语料倒进来。+pub struct FileFilter {+ include: Vec<glob::Pattern>,+ exclude: Vec<glob::Pattern>,+ max_file_bytes: i64,+}++impl FileFilter {+ pub fn from_config(config: &serde_json::Value) -> anyhow::Result<Self> {+ // 配置给了这一列就整列替换默认值+ let compile = |key: &str, defaults: &[&str]| -> anyhow::Result<Vec<glob::Pattern>> {+ let list: Vec<&str> = match config[key].as_array() {+ Some(a) => a.iter().filter_map(|v| v.as_str()).collect(),+ None => defaults.to_vec(),+ };+ list.into_iter()+ .map(|p| {+ glob::Pattern::new(p)+ .map_err(|e| anyhow::anyhow!("config.{key} has an invalid glob {p:?}: {e}"))+ })+ .collect()+ };+ Ok(Self {+ include: compile("include", &["**/*.md", "**/*.markdown", "**/*.mdx"])?,+ exclude: compile(+ "exclude",+ &["**/.*/**", "**/node_modules/**", "**/vendor/**"],+ )?,+ max_file_bytes: config["max_file_bytes"].as_i64().unwrap_or(1 << 20),+ })+ }++ fn eligible(&self, e: &TreeEntry) -> bool {+ let opts = glob::MatchOptions {+ case_sensitive: true,+ require_literal_separator: true,+ require_literal_leading_dot: false,+ };+ // 子模块是 commit、目录是 tree,符号链接指向别处——都不是文件本体+ e.kind == "blob"+ && e.mode != "120000"+ && e.size.is_some_and(|s| s <= self.max_file_bytes)+ && self.include.iter().any(|p| p.matches_with(&e.path, opts))+ && !self.exclude.iter().any(|p| p.matches_with(&e.path, opts))+ }+}++/// 按路径排序、同一 blob 只留第一条:相同文件在几个路径出现时,文档不该在路径之间横跳。+pub fn choose(entries: Vec<TreeEntry>, filter: &FileFilter) -> Vec<TreeEntry> {+ let mut entries: Vec<_> = entries.into_iter().filter(|e| filter.eligible(e)).collect();+ entries.sort_by(|a, b| a.path.cmp(&b.path));+ let mut seen = HashSet::new();+ entries.retain(|e| seen.insert(e.sha.clone()));+ entries+}++/// git 里一个 blob 的 SHA-1(先喂 `blob <字节数>\0` 再喂正文)。树里给的就是这个值,+/// 拿已存的字节算一遍就知道没变,不必再下文件。+pub fn git_blob_sha(bytes: &[u8]) -> String {+ let mut h = sha1::Sha1::new();+ h.update(format!("blob {}\0", bytes.len()).as_bytes());+ h.update(bytes);+ h.finalize().iter().map(|b| format!("{b:02x}")).collect()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn the_default_globs_take_markdown_outside_hidden_and_vendored_folders() {+ let f = FileFilter::from_config(&serde_json::json!({})).unwrap();+ let entry = |path: &str, mode: &str| TreeEntry {+ path: path.into(),+ mode: mode.into(),+ kind: "blob".into(),+ sha: "s".into(),+ size: Some(10),+ };+ for path in ["README.md", "docs/a/b.mdx"] {+ assert!(f.eligible(&entry(path, "100644")), "{path}");+ }+ for path in [+ ".github/PULL_REQUEST_TEMPLATE.md",+ "web/node_modules/x/README.md",+ "vendor/a.md",+ "Cargo.toml",+ ] {+ assert!(!f.eligible(&entry(path, "100644")), "{path}");+ }+ assert!(!f.eligible(&entry("link.md", "120000")));+ }+}diff --git a/crates/utopia-server/src/github_repo_sync_tests.rs b/crates/utopia-server/src/github_repo_sync_tests.rsnew file mode 100644index 00000000..9ce35c92--- /dev/null+++ b/crates/utopia-server/src/github_repo_sync_tests.rs@@ -0,0 +1,147 @@+//! 用录下的真实 GitHub 响应跑仓库同步:筛选、同一 blob 只收一处、没变不再取、+//! 树截断时不判「不在来源中」。+use super::sync_github_repo_with_client;+use serde_json::{json, Value};+use std::sync::{Arc, Mutex};+use uuid::Uuid;+use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate};++const PACKS: &str = "crates/utopia-server/packs/README.md";+const TESTDATA: &str = "crates/utopia-server/testdata/README.md";++/// 录下的树再加一个 testdata/README.md 的副本。阶段 1、2 去掉 packs,阶段 1 还标了截断+fn tree(fixture: &Value, stage: u8) -> Value {+ let mut entries = fixture["tree"]["tree"].as_array().unwrap().clone();+ let mut copy = entries+ .iter()+ .find(|e| e["path"] == TESTDATA)+ .unwrap()+ .clone();+ copy["path"] = json!("docs/copy/README.md");+ entries.push(copy);+ if stage > 0 {+ entries.retain(|e| e["path"] != PACKS);+ }+ json!({ "tree": entries, "truncated": stage == 1 })+}++#[tokio::test]+async fn a_repository_sync_takes_each_changed_file_once() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = sqlx::PgPool::connect(&url).await?;+ let (org, ws, kb, sid) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ let result = async {+ // 2000 字节的上限把 2920 字节的 SECURITY.md 挡在外面+ sqlx::raw_sql(&format!(+ r#"INSERT INTO organizations(id,name) VALUES('{org}','repo-test');+ INSERT INTO workspaces(id,org_id,name) VALUES('{ws}','{org}','repo-test');+ INSERT INTO knowledge_bases(id,workspace_id,name) VALUES('{kb}','{ws}','repo-test');+ INSERT INTO sources(id,kb_id,kind,name,config)+ VALUES('{sid}','{kb}','github_repo','fixture',+ '{{"repo":"deeplethe/utopia","max_file_bytes":2000}}');"#+ ))+ .execute(&pool)+ .await?;+ let source = utopia_store::sources::get(&pool, sid).await?;+ let dir = tempfile::tempdir()?;+ let cfg = utopia_core::config::AppConfig {+ data_dir: dir.path().to_string_lossy().into_owned(),+ ..Default::default()+ };+ let search = Arc::new(utopia_search::SearchIndex::open(+ &dir.path().join("search"),+ )?);+ let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into());++ let fixture: Value =+ serde_json::from_str(include_str!("../tests/fixtures/github_repo.json"))?;+ let server = MockServer::start().await;+ let stage = Arc::new(Mutex::new(0u8));+ let (current, fx) = (stage.clone(), fixture.clone());+ Mock::given(method("GET"))+ .respond_with(move |request: &Request| {+ let path = request.url.path();+ let body = if path.ends_with("/git/trees/HEAD") {+ tree(&fx, *current.lock().unwrap())+ } else if path.ends_with("/commits") {+ let (_, of) = request+ .url+ .query_pairs()+ .find(|(k, _)| k == "path")+ .unwrap();+ fx["commits"][&*of].clone()+ } else if let Some((_, sha)) = path.split_once("/git/blobs/") {+ return ResponseTemplate::new(200)+ .set_body_string(fx["blobs"][sha].as_str().unwrap());+ } else {+ return ResponseTemplate::new(404);+ };+ ResponseTemplate::new(200).set_body_json(body)+ })+ .mount(&server)+ .await;+ let (http, uri) = (reqwest::Client::builder().no_proxy().build()?, server.uri());+ let sync = || sync_github_repo_with_client(&state, &source, &http, &uri);+ let doc = |path: &str| {+ let key = format!("github:deeplethe/utopia:{path}");+ let pool = pool.clone();+ async move { utopia_store::documents::find_by_external_key(&pool, sid, &key).await }+ };++ let first = sync().await?;+ for path in [PACKS, TESTDATA] {+ let date = fixture["commits"][path][0]["commit"]["committer"]["date"].as_str();+ let time = doc(path).await?.and_then(|d| d.doc_time);+ anyhow::ensure!(+ time == date.and_then(|d| d.parse().ok()),+ "{path}:{time:?}"+ );+ }+ for path in [+ "SECURITY.md",+ ".github/PULL_REQUEST_TEMPLATE.md",+ "docs/copy/README.md",+ ] {+ anyhow::ensure!(doc(path).await?.is_none(), "{path} 不该收");+ }+ anyhow::ensure!(first.created == 2, "第一轮建了 {}", first.created);++ let seen = server.received_requests().await.unwrap().len();+ let second = sync().await?;+ let refetched = server.received_requests().await.unwrap()[seen..]+ .iter()+ .filter(|r| !r.url.path().ends_with("/git/trees/HEAD"))+ .count();+ anyhow::ensure!(+ second.total() == 0 && refetched == 0,+ "没变又取了 {refetched} 次"+ );++ *stage.lock().unwrap() = 1;+ sync().await?;+ let missing = || async { anyhow::Ok(doc(PACKS).await?.unwrap().missing_since) };+ anyhow::ensure!(missing().await?.is_none(), "截断的树不该判消失");+ *stage.lock().unwrap() = 2;+ sync().await?;+ anyhow::ensure!(missing().await?.is_some(), "完整的树里没了才判消失");+ anyhow::ensure!(doc(TESTDATA).await?.unwrap().missing_since.is_none());+ anyhow::Ok(())+ }+ .await;+ sqlx::query("DELETE FROM knowledge_bases WHERE id=$1")+ .bind(kb)+ .execute(&pool)+ .await?;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ result+}diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rsindex 17469be0..26e64209 100644--- a/crates/utopia-server/src/ingest_sources.rs+++ b/crates/utopia-server/src/ingest_sources.rs@@ -83,6 +83,7 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<()Some(SourceKind::Rss) => sync_rss(state, &source).await,Some(SourceKind::Custom) => sync_custom(state, &source, since).await,Some(SourceKind::GithubIssues) => sync_github_issues(state, &source, since).await,+ Some(SourceKind::GithubRepo) => sync_github_repo(state, &source).await,Some(SourceKind::JiraIssues) => sync_jira_issues(state, &source, since).await,Some(SourceKind::S3 | SourceKind::AzureBlob | SourceKind::Gcs) => {sync_object_storage(state, &source).await@@ -886,6 +887,106 @@ async fn sync_github_issues_with_client(Ok(stats)}+/// GitHub 仓库文件:一次递归取默认分支的整棵树,一条路径一篇文档。+/// 树里的 blob SHA 就判得出「没变」,变了的才逐条取提交日期与原文。+async fn sync_github_repo(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {+ // 与 github_issues 同一个口径:固定的公网主机按内容级严格度收+ let url = reqwest::Url::parse("https://api.github.com/")?;+ let http = crate::http_fetch::client_for(+ &url,+ crate::http_fetch::Reach::Content,+ crate::http_fetch::Limits::default(),+ )+ .await?;+ sync_github_repo_with_client(state, source, &http, "https://api.github.com").await+}++// HTTP 接缝让回归测试走同一条同步与落库路径,生产仍固定访问 GitHub 公网 API。+async fn sync_github_repo_with_client(+ state: &AppState,+ source: &Source,+ http: &reqwest::Client,+ api_base: &str,+) -> anyhow::Result<SyncStats> {+ use crate::{github_issues::get_checked, github_repo as gr};+ let repo = source.config["repo"]+ .as_str()+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .ok_or_else(|| anyhow::anyhow!("github_repo source is missing config.repo (owner/name)"))?;+ if repo.split('/').count() != 2 {+ anyhow::bail!("config.repo should look like owner/name, got {repo:?}");+ }+ let auth = source.config["auth_header"]+ .as_str()+ .map(str::trim)+ .filter(|s| !s.is_empty());+ let filter = gr::FileFilter::from_config(&source.config)?;+ let base = format!("{api_base}/repos/{repo}");+ let tree_url = reqwest::Url::parse(&format!("{base}/git/trees/HEAD?recursive=1"))?;+ let tree: gr::TreeResponse = get_checked(http, tree_url, auth, None)+ .await?+ .json()+ .await?;+ let (truncated, chosen) = (tree.truncated, gr::choose(tree.tree, &filter));+ let keys: Vec<String> = chosen+ .iter()+ .map(|e| format!("github:{repo}:{}", e.path))+ .collect();+ let mut stats = SyncStats::default();+ for (entry, key) in chosen.iter().zip(&keys) {+ // 没变的整条跳过:已落库的字节算出同一个 git SHA,就不必再花两次请求+ let old =+ utopia_store::documents::find_by_external_key(&state.pool, source.id, key).await?;+ let unchanged = match old {+ Some(d) if Some(d.size_bytes) == entry.size => state+ .blob+ .get(&d.sha256)+ .await+ .is_ok_and(|b| gr::git_blob_sha(&b) == entry.sha),+ _ => false,+ };+ if unchanged {+ continue;+ }+ // 文档日期取最后一次动过这条路径的提交,只在新出现或变了时问+ let mut cu = reqwest::Url::parse(&format!("{base}/commits"))?;+ cu.query_pairs_mut()+ .append_pair("path", &entry.path)+ .append_pair("per_page", "1");+ let commits: serde_json::Value = get_checked(http, cu, auth, None).await?.json().await?;+ let doc_time = commits[0]["commit"]["committer"]["date"]+ .as_str()+ .and_then(|d| d.parse().ok());+ let bu = reqwest::Url::parse(&format!("{base}/git/blobs/{}", entry.sha))?;+ let bytes = get_checked(http, bu, auth, Some("application/vnd.github.raw+json"))+ .await?+ .bytes()+ .await?;+ let action = ingest_item(+ state,+ source.kb_id,+ source.id,+ key,+ &entry.path,+ "text/markdown",+ &bytes,+ doc_time,+ )+ .await?;+ stats.absorb(action);+ }+ if truncated {+ // 树不全就不判「谁消失了」——缺席可能只是还没列到+ tracing::warn!(%repo, "GitHub 文件树被截断,本轮只摄入列出的文件,不做不在来源中对账");+ } else {+ utopia_store::documents::reconcile_missing(&state.pool, source.id, &keys)+ .await+ .map_err(|e| anyhow::anyhow!(e))?;+ }+ Ok(stats)+}+/// Jira 工单:一张工单一篇文档,正文里带**字段级**的变更史。////// 比 GitHub 那条路省:`search?expand=changelog` 一次调用就带回工单本体、@@ -1311,3 +1412,7 @@ mod source_filename_tests;#[cfg(test)]#[path = "github_sync_tests.rs"]mod github_sync_tests;++#[cfg(test)]+#[path = "github_repo_sync_tests.rs"]+mod github_repo_sync_tests;diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex 6617ea07..3ad724e3 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -10,6 +10,7 @@ mod error;mod extraction;mod extraction_open;mod github_issues;+mod github_repo;mod governance;mod http_fetch;mod implication;diff --git a/crates/utopia-server/tests/fixtures/github_repo.json b/crates/utopia-server/tests/fixtures/github_repo.jsonnew file mode 100644index 00000000..c420115e--- /dev/null+++ b/crates/utopia-server/tests/fixtures/github_repo.json@@ -0,0 +1,19 @@+{+ "note": "取自 deeplethe/utopia 的真实 GitHub 响应(树 cda3747),条目和字段已裁到我们读的那些。blobs 是 Accept: application/vnd.github.raw+json 取回的原文;commits 是 commits?path=…&per_page=1 的第一条。",+ "tree": {"sha": "cda374771577b2d42ed257312864ecf8e067bfbd", "truncated": false, "tree": [+ {"path": ".github/PULL_REQUEST_TEMPLATE.md", "mode": "100644", "type": "blob", "sha": "9b05a587001713b657e8cc51bb94f3a14add1dee", "size": 1067},+ {"path": "Cargo.toml", "mode": "100644", "type": "blob", "sha": "5b65de568922150141e12599d393e21db58eb6e3", "size": 4030},+ {"path": "SECURITY.md", "mode": "100644", "type": "blob", "sha": "274cff38310d7322ef84f27174e0344435ef5afc", "size": 2920},+ {"path": "crates/utopia-server/packs/README.md", "mode": "100644", "type": "blob", "sha": "69dbb3020bce31ed843185f1dd24af44184a5781", "size": 1364},+ {"path": "crates/utopia-server/testdata/README.md", "mode": "100644", "type": "blob", "sha": "2460624379264fed241f0986cd7dda7a5d86c104", "size": 1026},+ {"path": "docs", "mode": "040000", "type": "tree", "sha": "c2e443619c8b74e52530a296f1282b77c3e4f954"}+ ]},+ "commits": {+ "crates/utopia-server/packs/README.md": [{"sha": "92db693c3e4763f03551fcb1144684336ccd83f5", "commit": {"committer": {"date": "2026-08-30T01:17:00Z"}}}],+ "crates/utopia-server/testdata/README.md": [{"sha": "798fb2476f4ec8eacf3d6220657f27333ad50d94", "commit": {"committer": {"date": "2026-09-30T10:04:58Z"}}}]+ },+ "blobs": {+ "69dbb3020bce31ed843185f1dd24af44184a5781": "# 预制本体包\n\n来源文件按原样保存(gzip 压缩),由 `src/ontology_packs.rs` 用 `include_bytes!` 内嵌进二进制。\n\n**为什么内嵌而不是运行时下载**:README 承诺整套系统可以运行在完全离线的内网环境。\n运行时抓取会让这句话失效,也让构建不可复现(上游随时会发新版)。\n\n**为什么压缩**:未压缩合计 1.7 MB,压缩后 316 KB。这是个公开仓库,clone 体积是真实成本。\n`flate2` 本来就在依赖树里,解压是三行。\n\n| 文件 | 来源 | 许可 | 抓取日 |\n|---|---|---|---|\n| `schema-org.ttl.gz` | https://schema.org/version/latest/schemaorg-current-https.ttl | CC BY-SA 3.0 | 2026-08-30 |\n| `w3c-org.ttl.gz` | https://www.w3.org/ns/org.ttl | W3C Document License | 2026-08-30 |\n| `prov-o.ttl.gz` | https://www.w3.org/ns/prov.ttl | W3C Document License | 2026-08-30 |\n| `foaf.rdf.gz` | http://xmlns.com/foaf/spec/index.rdf | CC BY 1.0 | 2026-08-30 |\n| `iof-core.rdf.gz` | https://spec.industrialontologies.org/ontology/core/Core/ | MIT | 2026-08-30 |\n\n## 更新一个包\n\n重新抓取、`gzip -9c` 覆盖、更新 `ontology_packs.rs` 里的计数与本文件的抓取日。\n**不要改动原文内容**——投影只覆盖当下能消费的部分,原文保真是 [0001](../../../docs/decisions/0001-ontology-import-and-governance.md) 判据 1。\n",+ "2460624379264fed241f0986cd7dda7a5d86c104": "# OCR page fixture\n\n`ark-ocr-pages.pdf` is a synthetic three-page PDF created for OCR acceptance tests.\nPages 1 and 3 contain rasterized Chinese and English text, dates, amounts, a small\ntable, an instruction printed as source text, and an unlabelled line chart. Page 2\nis blank. There is no text layer. All content is authored for this test and carries\nno third-party or private data.\n\nThe fixture tests real Poppler rendering, original page numbering after a blank\npage, and restarting a failed pass from page 1. Mock replies test those contracts;\nthey do not measure a model's recognition accuracy.\n\nWith `ARK_API_KEY` set, run the manual Agent Plan acceptance test:\n\n```sh\ncargo test -p utopia-server live_agent_plan_reads_scans -- --ignored --nocapture\n```\n\nThis sends the probe and this fixture to the real API. Optional\n`ARK_OCR_TEST_PDF` adds a public scan, and `ARK_OCR_TEST_OUTPUT` saves its text and\nprovenance without credentials. `ARK_OCR_TEST_BASE_URL` and\n`ARK_OCR_TEST_MODEL` override the Agent Plan defaults.\n"+ }+}diff --git a/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs b/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rsindex 269f9feb..68674ec1 100644--- a/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs+++ b/crates/utopia-store/tests/store/a_viewer_never_sees_a_credential.rs@@ -56,6 +56,10 @@ async fn a_viewer_never_sees_a_credential() -> anyhow::Result<()> {"github_issues",serde_json::json!({ "repo": "o/r", "auth_header": "Bearer g" }),),+ (+ "github_repo",+ serde_json::json!({ "repo": "o/r", "auth_header": "Bearer r" }),+ ),("jira_issues",serde_json::json!({ "base_url": "https://j.test", "project": "P", "auth_header": "Basic j" }),diff --git a/docs/design/sources.md b/docs/design/sources.mdindex bbc34da1..d543cec1 100644--- a/docs/design/sources.md+++ b/docs/design/sources.md@@ -19,7 +19,9 @@ document date, so an upload-dated document counts as undated for attestation andweb list derive from; credentials never leave the server (`SOURCE_SECRET_KEYS`) and a blank fieldkeeps the stored value; a truncated round says so [0013]. A ticket is one document whose bodycarries its history as dated declarative sentences and a `## History` list; GitHub fetches events-per ticket; timestamps are written to the second [0013].+per ticket; timestamps are written to the second [0013]. A GitHub repository file is one document+keyed by its path and dated by the last commit that touched it; a file whose blob SHA has not+changed is not fetched again, and identical files at several paths go in once [#1303].**Statements.** A `statements` source takes the open extraction contract itself (`e`, `s`, `n`) on `POST /sources/{id}/statements`, with the `api` push's identity, versions, tombstones and run history; the payload is the document and its single chunk, and extraction parses the chunk instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement with the item as its evidence (null offsets) and reaches the typed graph only through alignment. The contract has no slot for a property or a class and unknown keys are refused; a second push under the same identity marks the earlier statements stale rather than closing them; tables belong on a mount, not here [0054].diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex a7109fcb..71284011 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -640,6 +640,7 @@ export const en = {statements: "Statements",custom: "Custom",github_issues: "GitHub issues",+ github_repo: "GitHub repository files",jira_issues: "Jira issues",s3: "S3 / MinIO",azure_blob: "Azure Blob",@@ -659,6 +660,9 @@ export const en = {"Syncs a repository's issues. **Each ticket, together with its state history**, becomes " +"one document — when it was opened, closed, relabelled, reassigned, all dated. " +"Without a token GitHub allows only 60 requests an hour.",+ github_repo:+ "Reads markdown files off a repository's default branch. Each file becomes one " ++ "document dated by its last commit; unchanged files are not fetched again.",s3:"Reads documents out of an S3 bucket, or anything speaking the same protocol " +"(MinIO, Ceph, R2). **Leave the endpoint empty for AWS**; fill it in for a " +@@ -756,6 +760,8 @@ export const en = {notionQueryField: "Search term (optional — empty takes every shared page)",tokenField: "GitHub token (optional, never shown again)",includePullRequests: "Treat pull requests as tickets too",+ includeGlobsField: "Include globs (comma-separated, optional)",+ excludeGlobsField: "Exclude globs (comma-separated, optional)",interval: "Sync schedule",intervalManual: "Manual only",intervalEvery: (m: number) =>diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 9222d4ed..e7485665 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -579,6 +579,7 @@ export const zh: Strings = {statements: "陈述",custom: "自定义",github_issues: "GitHub 工单",+ github_repo: "GitHub 仓库文件",jira_issues: "Jira 工单",s3: "S3 / MinIO",azure_blob: "Azure Blob",@@ -597,6 +598,9 @@ export const zh: Strings = {"同步一个仓库的工单。**每张工单连同它的状态变更史**成为一篇文档——" +"何时开出、何时关闭、标签与负责人怎么变的,都带着日期。未配令牌时" +"每小时只有 60 次请求配额。",+ github_repo:+ "读取一个仓库默认分支上的 markdown 文件。每个文件成为一篇文档," ++ "日期取最后一次改动它的提交;没变的文件不会重抓。",s3:"从 S3 桶里读文档,或者任何说同一套协议的东西(MinIO、Ceph、R2)。" +"**端点留空就是 AWS**,填了就是自建。每个对象成为一篇文档," +@@ -689,6 +693,8 @@ export const zh: Strings = {notionQueryField: "搜索词(可选——留空则取全部已分享页面)",tokenField: "GitHub 令牌(可选,不再显示)",includePullRequests: "把 PR 也当工单收进来",+ includeGlobsField: "包含的 glob(逗号分隔,可选)",+ excludeGlobsField: "排除的 glob(逗号分隔,可选)",interval: "同步计划",intervalManual: "仅手动",intervalEvery: (m: number) =>diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsxindex 3fe65b45..40e314a0 100644--- a/web/src/pages/Library.tsx+++ b/web/src/pages/Library.tsx@@ -1360,6 +1360,8 @@ function SourceModal({// PR 在 GitHub 的模型里也是工单。默认不收——问「工单」要的是工单;// 但有些仓库的决策记录实际写在 PR 描述里,所以给个开关const [includePrs, setIncludePrs] = useState(false);+ const [includeGlobs, setIncludeGlobs] = useState("");+ const [excludeGlobs, setExcludeGlobs] = useState("");const [schedule, setSchedule] = useState<ScheduleValue>({sync_interval_minutes: null,sync_cron: null,@@ -1370,6 +1372,7 @@ function SourceModal({kind === "rss" ||kind === "custom" ||kind === "github_issues" ||+ kind === "github_repo" ||kind === "jira_issues" ||kind === "s3" ||kind === "azure_blob" ||@@ -1379,6 +1382,11 @@ function SourceModal({const create = useMutation({mutationFn: () => {+ // 留空就不发,后端按默认的 markdown 规则筛+ const globs = (key: string, value: string) => {+ const list = value.split(",").map((g) => g.trim()).filter(Boolean);+ return list.length ? { [key]: list } : {};+ };const config =kind === "url"? { urls: urls.split("\n").map((u) => u.trim()).filter(Boolean) }@@ -1389,11 +1397,13 @@ function SourceModal({endpoint: endpoint.trim(),...(authHeader.trim() ? { auth_header: authHeader.trim() } : {}),}- : kind === "github_issues"+ : kind === "github_issues" || kind === "github_repo"? {repo: repo.trim(),...(authHeader.trim() ? { auth_header: authHeader.trim() } : {}),- ...(includePrs ? { include_pull_requests: true } : {}),+ ...(kind === "github_issues"+ ? includePrs ? { include_pull_requests: true } : {}+ : { ...globs("include", includeGlobs), ...globs("exclude", excludeGlobs) }),}: kind === "jira_issues"? {@@ -1799,7 +1809,7 @@ function SourceModal({)}</>)}- {kind === "github_issues" && (+ {(kind === "github_issues" || kind === "github_repo") && (<>{field(S.library.repoField,@@ -1818,12 +1828,33 @@ function SourceModal({onChange={(e) => setAuthHeader(e.target.value)}/>,)}- <Checkbox- className="mb-4"- checked={includePrs}- onChange={(v) => setIncludePrs(v)}- label={S.library.includePullRequests}- />+ {kind === "github_issues" ? (+ <Checkbox+ className="mb-4"+ checked={includePrs}+ onChange={(v) => setIncludePrs(v)}+ label={S.library.includePullRequests}+ />+ ) : (+ <>+ {field(+ S.library.includeGlobsField,+ <Input className="w-full font-mono"+ placeholder="**/*.md, **/*.markdown, **/*.mdx"+ value={includeGlobs}+ onChange={(e) => setIncludeGlobs(e.target.value)}+ />,+ )}+ {field(+ S.library.excludeGlobsField,+ <Input className="w-full font-mono"+ placeholder="**/.*/**, **/node_modules/**, **/vendor/**"+ value={excludeGlobs}+ onChange={(e) => setExcludeGlobs(e.target.value)}+ />,+ )}+ </>+ )}</>)}diff --git a/web/src/pages/SourcesRail.tsx b/web/src/pages/SourcesRail.tsxindex 9929bb0b..e881dfba 100644--- a/web/src/pages/SourcesRail.tsx+++ b/web/src/pages/SourcesRail.tsx@@ -72,6 +72,7 @@ export const KIND_ICON: Record<SourceKind, LucideIcon> = {statements: Braces,custom: Puzzle,github_issues: CircleDot,+ github_repo: FileText,jira_issues: SquareKanban,s3: HardDrive,azure_blob: Cloud,@@ -88,6 +89,7 @@ export const SYNCING_KINDS = new Set(["rss","custom","github_issues",+ "github_repo","jira_issues","s3","azure_blob",diff --git a/web/src/sourceKinds.ts b/web/src/sourceKinds.tsindex c6e30af4..35c82f18 100644--- a/web/src/sourceKinds.ts+++ b/web/src/sourceKinds.ts@@ -13,6 +13,7 @@ export const CREATABLE_SOURCE_KINDS = ["url","rss","github_issues",+ "github_repo","jira_issues","s3","azure_blob",