World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A bucket is a source, and MinIO speaks the same protocol (#200)
WaylandYanga14a37051621
diff --git a/Cargo.lock b/Cargo.lockindex 1710a8ee..66810570 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -707,6 +707,16 @@ version = "2.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"+[[package]]+name = "crc-fast"+version = "1.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5"+dependencies = [+ "digest 0.10.7",+ "spin 0.10.1",+]+[[package]]name = "crc32fast"version = "1.5.1"@@ -1298,7 +1308,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"dependencies = ["futures-core","futures-sink",- "spin",+ "spin 0.9.9",][[package]]@@ -1659,6 +1669,12 @@ version = "1.0.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"+[[package]]+name = "humantime"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"+[[package]]name = "hybrid-array"version = "0.4.14"@@ -2002,6 +2018,15 @@ dependencies = ["either",]+[[package]]+name = "itertools"+version = "0.15.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"+dependencies = [+ "either",+]+[[package]]name = "itoa"version = "1.0.18"@@ -2127,7 +2152,7 @@ version = "1.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"dependencies = [- "spin",+ "spin 0.9.9",][[package]]@@ -2237,7 +2262,7 @@ dependencies = ["indexmap","itoa","log",- "md-5",+ "md-5 0.10.6","nom 8.0.0","rand 0.10.2","rangemap",@@ -2305,6 +2330,16 @@ dependencies = ["digest 0.10.7",]+[[package]]+name = "md-5"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"+dependencies = [+ "cfg-if",+ "digest 0.11.3",+]+[[package]]name = "measure_time"version = "0.9.0"@@ -2394,7 +2429,7 @@ dependencies = ["httparse","memchr","mime",- "spin",+ "spin 0.9.9","version_check",]@@ -2410,6 +2445,18 @@ version = "1.0.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"+[[package]]+name = "nix"+version = "0.31.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"+dependencies = [+ "bitflags",+ "cfg-if",+ "cfg_aliases",+ "libc",+]+[[package]]name = "no_std_io2"version = "0.9.4"@@ -2517,6 +2564,48 @@ dependencies = ["memchr",]+[[package]]+name = "object_store"+version = "0.14.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50"+dependencies = [+ "async-trait",+ "aws-lc-rs",+ "base64",+ "bytes",+ "chrono",+ "crc-fast",+ "form_urlencoded",+ "futures-channel",+ "futures-core",+ "futures-util",+ "http",+ "http-body-util",+ "humantime",+ "hyper",+ "itertools 0.15.0",+ "md-5 0.11.0",+ "nix",+ "parking_lot",+ "percent-encoding",+ "quick-xml 0.41.0",+ "rand 0.10.2",+ "reqwest",+ "rustls-pki-types",+ "serde",+ "serde_json",+ "serde_urlencoded",+ "thiserror 2.0.20",+ "tokio",+ "tracing",+ "url",+ "walkdir",+ "wasm-bindgen-futures",+ "web-time",+ "windows-sys 0.61.2",+]+[[package]]name = "once_cell"version = "1.21.4"@@ -2970,6 +3059,7 @@ checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"dependencies = ["encoding_rs","memchr",+ "serde",][[package]]@@ -3739,6 +3829,12 @@ dependencies = ["lock_api",]+[[package]]+name = "spin"+version = "0.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"+[[package]]name = "spki"version = "0.7.3"@@ -3874,7 +3970,7 @@ dependencies = ["hmac","itoa","log",- "md-5",+ "md-5 0.10.6","memchr","once_cell","percent-encoding",@@ -3915,7 +4011,7 @@ dependencies = ["home","itoa","log",- "md-5",+ "md-5 0.10.6","memchr","once_cell","rand 0.8.8",@@ -4128,7 +4224,7 @@ dependencies = ["fnv","fs4","htmlescape",- "itertools",+ "itertools 0.14.0","levenshtein_automata","log","lru",@@ -4177,7 +4273,7 @@ checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc"dependencies = ["downcast-rs","fastdivide",- "itertools",+ "itertools 0.14.0","serde","tantivy-bitpacker","tantivy-common",@@ -4240,7 +4336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606"dependencies = ["futures-util",- "itertools",+ "itertools 0.14.0","tantivy-bitpacker","tantivy-common","tantivy-fst",@@ -4299,7 +4395,7 @@ dependencies = ["either","icu_provider","icu_segmenter",- "itertools",+ "itertools 0.14.0","memchr","strum","thiserror 2.0.20",@@ -4831,6 +4927,7 @@ dependencies = ["flate2","futures-util","jsonwebtoken",+ "object_store","reqwest","serde","serde_json",diff --git a/crates/utopia-server/Cargo.toml b/crates/utopia-server/Cargo.tomlindex f4acec6d..a521ab5c 100644--- a/crates/utopia-server/Cargo.toml+++ b/crates/utopia-server/Cargo.toml@@ -39,3 +39,4 @@ reqwest.workspace = truefeed-rs.workspace = trueasync-trait = "0.1.92"sqlparser = "0.62.0"+object_store = { version = "0.14.1", features = ["aws"] }diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rsindex 890e2964..3f8ecd1c 100644--- a/crates/utopia-server/src/ingest_sources.rs+++ b/crates/utopia-server/src/ingest_sources.rs@@ -1,5 +1,6 @@//! 来源同步任务:url(网页抓取)/ rss(订阅,pubDate → doc_time)/-//! github_issues / jira_issues(工单,更新时刻 → doc_time,正文里带状态变更史)。+//! github_issues / jira_issues(工单,更新时刻 → doc_time,正文里带状态变更史)/+//! s3(对象存储,LastModified → doc_time)。//! 全部走 sha256 去重(重复内容静默跳过),新文档进标准摄入管道(process_document)。//! folder 是纯容器(上传入内),api 是推送型——两者无拉取语义。@@ -53,6 +54,7 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<()"custom" => sync_custom(state, &source).await,"github_issues" => sync_github_issues(state, &source).await,"jira_issues" => sync_jira_issues(state, &source).await,+ "s3" => sync_object_storage(state, &source).await,// folder / api 无拉取语义_ => Ok(SyncStats::default()),};@@ -746,6 +748,61 @@ async fn sync_custom(state: &AppState, source: &Source) -> anyhow::Result<SyncStOk(stats)}+/// 对象存储同步:列前缀下的对象,逐个摄入。+///+/// `external_key` 用 `s3://bucket/key`,跟 `file:///` 与页面 URL 同一个约定:+/// 出处自描述。换了前缀但内容没变时,`ingest_item` 会认成「搬家」而不是新增,+/// 不会重跑一遍抽取。+///+/// **`doc_time` 取 `LastModified`,而它是写入时刻不是文档自身的时间。**+/// 一份 2019 年的合同今天传上去,时间线上会落在今天。对象存储没有更好的+/// 来源——除非文件名或正文里带日期,而那是抽取器的活。这一条与 `url` 源+/// 同病:`0013` 的第一条判据在这里只满足一半。+async fn sync_object_storage(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {+ let bucket = source.config["bucket"]+ .as_str()+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .ok_or_else(|| anyhow::anyhow!("s3 source is missing config.bucket"))?;+ let prefix = source.config["prefix"]+ .as_str()+ .map(str::trim)+ .filter(|s| !s.is_empty());++ let store = crate::object_storage::client(&source.config)?;+ let (objects, truncated) = crate::object_storage::fetch(store.as_ref(), bucket, prefix).await?;++ // 到顶不是错误,但必须说出来——否则「同步成功」下面藏着没进来的东西,+ // 而那正是 0005 说的失败无声+ if truncated {+ tracing::warn!(+ %bucket, prefix = prefix.unwrap_or(""),+ "对象数到达单次上限,其余留给下一次同步"+ );+ }++ let mut stats = SyncStats::default();+ for obj in objects {+ // **不猜 mime。** `utopia_ingest::parse` 先看魔数、再看扩展名,+ // 注释写着「扩展名可能撒谎」——在这里按文件名猜一个,只是多一个+ // 会撒谎的来源,而且要为此多一个依赖。`octet-stream` 是诚实的:+ // 我们拿到的就是一串字节,没看过里面是什么。+ let action = ingest_item(+ state,+ source.kb_id,+ source.id,+ &obj.external_key,+ &obj.filename,+ "application/octet-stream",+ &obj.bytes,+ obj.last_modified,+ )+ .await?;+ stats.absorb(action);+ }+ Ok(stats)+}+/// 标题无扩展名时按 mime 补一个,接入解析矩阵的分派。fn ensure_extension(title: &str, mime: &str) -> String {let has_ext = titlediff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex 34217e0b..01613d88 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -14,6 +14,7 @@ mod jira_issues;mod live;mod llm_util;mod mappings;+mod object_storage;mod ontology_index;mod ontology_packs;mod owl_import;diff --git a/crates/utopia-server/src/object_storage.rs b/crates/utopia-server/src/object_storage.rsnew file mode 100644index 00000000..a4f381f0--- /dev/null+++ b/crates/utopia-server/src/object_storage.rs@@ -0,0 +1,275 @@+//! 对象存储来源:S3 与一切说同一套协议的东西(MinIO、Ceph RGW、阿里 OSS 的+//! S3 兼容端点、Cloudflare R2…)。+//!+//! 按 [0013](../../docs/decisions/0013-a-source-should-hand-over-its-history.md)+//! 的四条判据看,它跟工单系统不是一类:+//!+//! | 判据 | 对象存储 |+//! |---|---|+//! | 真实时间戳 | `LastModified`,但它是**写入时刻**不是文档自身的时间 |+//! | 会不会自我推翻 | 同 key 覆写就是一次改口,但要靠两次同步之间的差异看出来 |+//! | 稳定身份 | `bucket/key`,最干净的一条 |+//! | 企业知识住不住在那儿 | **最强的一条**——文档堆、归档、数据湖落地区都在这儿 |+//!+//! 所以它的价值在第四条:**够得着**。第二条只有部分——工单系统能一次交出+//! 整段变更史,而对象存储要么开了版本控制(本版不读,见下),要么只能靠+//! 一次次同步慢慢攒。这跟 `url` / `rss` 是同一档,不比它们低。+//!+//! **版本历史本版不做。** `ListObjectVersions` 能一次拿到 0013 想要的那段历史,+//! 但多数桶没开版本控制,开了的桶里也大多是写一次不再动的对象——把每个版本+//! 都摄进来,代价与收益不成比例。等有人真的需要(一份季度更新的制度文件,+//! 想看它改了什么)再按开关加。+//!+//! **为什么是 `object_store` 而不是 `aws-sdk-s3` / `rust-s3`**:实测传递依赖+//! `+7` 对 `+43` / `+42`——它复用了树里已有的 `reqwest`、`ring`、`quick-xml`。+//! 而且同一套 API 还覆盖 Azure Blob 与 GCS,路线图上「更多数据源」与+//! 「数据湖仓」两条都在这条线上,换后端只是换一个 builder。++use anyhow::Context as _;+use chrono::{DateTime, Utc};+use futures_util::StreamExt as _;+use object_store::aws::AmazonS3Builder;+// `get` 住在 `ObjectStoreExt` 上,不在 `ObjectStore` 本体——后者只有+// `get_opts`。少这一行的报错是「&dyn ObjectStore 没有 get 方法」,+// 而 trait 名字里看不出来。+use object_store::{path::Path as StorePath, ObjectStore, ObjectStoreExt as _};++/// 一次同步最多摄入多少个对象。+///+/// **不是性能考虑,是「别把一个桶整个吸进来」。** 桶里放上百万个对象是常态,+/// 而摄入是不可逆的:每个对象要抽取、要向量化、要进图。配错一个前缀就能+/// 烧掉一整天的额度,而且清理起来比配置麻烦得多。+///+/// 到顶时报进同步统计,让人看得见「还有」,而不是悄悄截断。+const MAX_OBJECTS_PER_SYNC: usize = 2_000;++/// 单个对象的大小上限。跟上传路径同一个数量级——超过这个尺寸的多半不是+/// 文档而是数据文件(备份、镜像、视频),抽取器拿它没有用。+const MAX_OBJECT_BYTES: u64 = 32 * 1024 * 1024;++/// 一个待摄入的对象。+pub struct RemoteObject {+ /// `s3://bucket/key`——`ingest_item` 的 external_key 约定是 URI 形态+ pub external_key: String,+ /// 落进文档的文件名,取 key 的最后一段+ pub filename: String,+ pub bytes: Vec<u8>,+ pub last_modified: Option<DateTime<Utc>>,+}++/// 从来源配置建一个客户端。+///+/// **`endpoint` 在则走自建**(MinIO、Ceph、R2),此时强制 path-style:+/// 虚拟主机式寻址要求 `bucket.host` 这样的域名解析得开,而自建部署通常+/// 只有一个裸 IP 或一个主机名。不强制的话症状是「桶不存在」,+/// 而桶明明就在那儿。+///+/// 允许 http 也是为它:内网 MinIO 常常不挂 TLS。**这是配置说了算的降级**,+/// 不是默认——公有云那条路(不给 endpoint)仍然只走 https。+pub fn client(config: &serde_json::Value) -> anyhow::Result<Box<dyn ObjectStore>> {+ let s = |k: &str| {+ config[k]+ .as_str()+ .map(str::trim)+ .filter(|v| !v.is_empty())+ .map(str::to_string)+ };+ let bucket = s("bucket")+ .ok_or_else(|| anyhow::anyhow!("object storage source is missing config.bucket"))?;++ let mut b = AmazonS3Builder::new().with_bucket_name(&bucket);+ if let Some(region) = s("region") {+ b = b.with_region(region);+ }+ if let (Some(key), Some(secret)) = (s("access_key_id"), s("secret_access_key")) {+ b = b.with_access_key_id(key).with_secret_access_key(secret);+ }+ if let Some(endpoint) = s("endpoint") {+ let insecure = endpoint.starts_with("http://");+ b = b+ .with_endpoint(endpoint)+ .with_virtual_hosted_style_request(false)+ .with_allow_http(insecure);+ }+ Ok(Box::new(b.build().context("object storage client")?))+}++/// 列出前缀下的对象并逐个取回。+///+/// **零字节的一律跳过,判据是大小不是名字。** 直觉的写法是看 key 以不以 `/`+/// 结尾——控制台和 `aws s3 sync` 就是这样造「文件夹」的。但 `object_store`+/// 的 `Path` 会**把尾斜杠规范化掉**,`docs/` 到手里就成了 `docs`,那个判断+/// 永远不成立;接着它去 GET `docs`,而桶里真实的 key 是 `docs/`,+/// 于是 404 把整次同步带走。实测在 MinIO 上就是这样炸的。+///+/// 按大小判还更宽:零字节的对象无论叫什么都不是文档,`ingest_item` 拿到+/// 空字节也只会返回 `Unchanged`。+///+/// 格式不在这里判。`utopia_ingest` 的抽取按扩展名加 mime 分派,认不出的+/// 一律按文本解码——在这里再维护一张白名单,就是同一件事写两遍,+/// 而两份清单迟早分叉。+pub async fn fetch(+ store: &dyn ObjectStore,+ bucket: &str,+ prefix: Option<&str>,+) -> anyhow::Result<(Vec<RemoteObject>, bool)> {+ let p = prefix.map(StorePath::from);+ let mut listing = store.list(p.as_ref());+ let mut out = Vec::new();+ let mut truncated = false;+ let mut unreadable = 0usize;++ while let Some(meta) = listing.next().await {+ let meta = meta.context("listing objects")?;+ let key = meta.location.as_ref().to_string();++ if meta.size == 0 {+ continue;+ }+ if meta.size > MAX_OBJECT_BYTES {+ tracing::info!(%key, size = meta.size, "对象太大,跳过");+ continue;+ }+ if out.len() >= MAX_OBJECTS_PER_SYNC {+ truncated = true;+ break;+ }++ // **一个对象取不回来不该带走整次同步。** 列表与取回之间隔着时间,+ // 中途被删、被换权限都是正常的;桶越大越常见。记下来在收尾时一并报,+ // 而不是让第 900 个对象的一次 404 把前 899 个的成果一起丢掉。+ let got = async {+ let r = store.get(&meta.location).await?;+ r.bytes().await+ }+ .await;+ let bytes = match got {+ Ok(b) => b,+ Err(e) => {+ unreadable += 1;+ tracing::warn!(%key, error = %e, "对象取不回来,跳过");+ continue;+ }+ };++ out.push(RemoteObject {+ external_key: format!("s3://{bucket}/{key}"),+ filename: key.rsplit('/').next().unwrap_or(&key).to_string(),+ bytes: bytes.to_vec(),+ last_modified: Some(meta.last_modified),+ });+ }+ if unreadable > 0 {+ tracing::warn!(unreadable, bucket, "有对象没能取回");+ }+ Ok((out, truncated))+}++#[cfg(test)]+mod tests {+ use super::*;++ /// 自建端点必须走 path-style,否则请求会打到 `bucket.host` 上,+ /// 而自建部署一般没有那条 DNS 记录。症状是「桶不存在」而桶就在那儿。+ #[test]+ fn a_self_hosted_endpoint_builds() {+ let cfg = serde_json::json!({+ "bucket": "docs",+ "endpoint": "http://127.0.0.1:9000",+ "region": "us-east-1",+ "access_key_id": "minioadmin",+ "secret_access_key": "minioadmin",+ });+ assert!(client(&cfg).is_ok());+ }++ /// 不给 bucket 要报得出是缺哪一项——照 jira / github 两个来源的措辞。+ #[test]+ fn a_missing_bucket_says_so() {+ let e = client(&serde_json::json!({ "region": "us-east-1" })).unwrap_err();+ assert!(e.to_string().contains("config.bucket"), "{e}");+ }++ /// 公有云那条路:不给 endpoint 就不该被强制成 path-style,+ /// 也不该允许明文 http。+ #[test]+ fn a_cloud_bucket_needs_no_endpoint() {+ let cfg = serde_json::json!({+ "bucket": "docs",+ "region": "ap-southeast-1",+ "access_key_id": "AKIA...",+ "secret_access_key": "...",+ });+ assert!(client(&cfg).is_ok());+ }++ /// 真连一个 S3 兼容端点,只测**读**这一侧:列出来、取回内容、身份对不对。+ ///+ /// **没有 `UTOPIA_S3_TEST_ENDPOINT` 时跳过而不是失败**——跟连库的测试+ /// 同一个约定(见 CONTRIBUTING)。上面那三条只证明 builder 拼得出来,+ /// 而这条要守的是**线上协议真的说得通**:path-style 有没有生效、+ /// 明文 http 放没放行、`s3://` 身份拼得对不对、`LastModified` 回不回来。+ /// 没有一条是构造 client 时能发现的。+ ///+ /// **数据由外部布置,测试不写只读。** 第一版让测试自己 `put`,结果被+ /// `object_store` 的 `Path` 咬了一口:它会把 `docs/` 规范化成 `docs`,+ /// 于是「目录占位」写成了一个名叫 `docs` 的普通对象,而 MinIO 不允许+ /// 对象 `docs` 与前缀 `docs/` 并存——整个前缀被顶掉,列表回来是空的。+ /// 真实的占位对象只能由别的客户端造,那就让它由别的客户端造。+ ///+ /// 跑法(先起 MinIO,再用 curl 布数据;curl 自带 SigV4):+ /// ```text+ /// MINIO_ROOT_USER=u MINIO_ROOT_PASSWORD=p123456 \+ /// minio server ./data --address 127.0.0.1:19000 &+ /// S3="curl -s --aws-sigv4 aws:amz:us-east-1:s3 -u u:p123456"+ /// B=http://127.0.0.1:19000/utopia-conn-test+ /// $S3 -X PUT $B # 建桶+ /// $S3 -X PUT --data-raw hello "$B/docs/a.txt"+ /// $S3 -X PUT --data-raw '# world' "$B/docs/b.md"+ /// $S3 -X PUT --data-raw '' "$B/docs/" # 目录占位+ /// UTOPIA_S3_TEST_ENDPOINT=http://127.0.0.1:19000 \+ /// UTOPIA_S3_TEST_KEY=u UTOPIA_S3_TEST_SECRET=p123456 \+ /// cargo test -p utopia-server object_storage+ /// ```+ #[tokio::test]+ async fn it_reads_from_a_real_s3_endpoint() -> anyhow::Result<()> {+ let Ok(endpoint) = std::env::var("UTOPIA_S3_TEST_ENDPOINT") else {+ eprintln!("跳过:未设 UTOPIA_S3_TEST_ENDPOINT");+ return Ok(());+ };+ let bucket = "utopia-conn-test";+ let cfg = serde_json::json!({+ "bucket": bucket,+ "endpoint": endpoint,+ "region": "us-east-1",+ "access_key_id": std::env::var("UTOPIA_S3_TEST_KEY").unwrap_or("minioadmin".into()),+ "secret_access_key": std::env::var("UTOPIA_S3_TEST_SECRET").unwrap_or("minioadmin".into()),+ });+ let store = client(&cfg)?;++ let (objs, truncated) = fetch(store.as_ref(), bucket, Some("docs")).await?;+ assert!(!truncated, "三个对象不该触发上限");++ let keys: Vec<&str> = objs.iter().map(|o| o.external_key.as_str()).collect();+ for want in [+ "s3://utopia-conn-test/docs/a.txt",+ "s3://utopia-conn-test/docs/b.md",+ ] {+ assert!(keys.contains(&want), "少了 {want}:{keys:?}");+ }++ // **这条守的是一个真炸过的 bug。** 桶里有一个 `docs/` 目录占位对象,+ // 而 `object_store` 把尾斜杠规范化掉了,于是它以 `docs` 的身份进到+ // 循环里,GET 回来 404,整次同步失败。三条构造 client 的单元测试+ // 一条都发现不了——只有真连一个装过「文件夹」的桶才看得见。+ assert_eq!(objs.len(), 2, "目录占位对象没被滤掉:{keys:?}");++ let a = objs.iter().find(|o| o.filename == "a.txt").expect("a.txt");+ assert_eq!(a.bytes, b"hello", "取回的内容不对");+ assert!(+ a.last_modified.is_some(),+ "LastModified 是 doc_time 的唯一来源"+ );+ Ok(())+ }+}diff --git a/web/src/api.ts b/web/src/api.tsindex 32d364e7..4be615e5 100644--- a/web/src/api.ts+++ b/web/src/api.ts@@ -169,6 +169,7 @@ export interface SourceView {| "custom"| "github_issues"| "jira_issues"+ | "s3"| "memory"| "upload";name: string;diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex a133c070..45c260c9 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -418,6 +418,7 @@ export const en = {custom: "Custom",github_issues: "GitHub issues",jira_issues: "Jira issues",+ s3: "Object storage",},sourceKindHints: {folder:@@ -432,6 +433,10 @@ export const en = {"Syncs a repository's issues. **Each ticket, together with its state history**, becomes " +"one document — when it was opened, closed, relabelled, reassigned, all dated. " +"Without a token GitHub allows only 60 requests an hour.",+ s3:+ "Reads documents out of an S3 bucket, or anything speaking the same protocol " ++ "(MinIO, Ceph, R2). **Leave the endpoint empty for AWS**; fill it in for a " ++ "self-hosted one. Each object becomes a document dated by its last modification.",api: "External systems push JSON documents here, authenticated with this source's own token.",custom:"Polls a URL you control on a schedule — your service returns JSON items and Utopia keeps them in sync.",@@ -478,6 +483,12 @@ export const en = {repoField: "Repository (owner/name)",jiraUrlField: "Jira site URL",jiraProjectField: "Project key",+ s3BucketField: "Bucket",+ s3PrefixField: "Prefix (optional — without one the whole bucket is read)",+ s3EndpointField: "Endpoint (leave empty for AWS S3)",+ s3RegionField: "Region",+ s3KeyField: "Access key ID",+ s3SecretField: "Secret access key",tokenField: "GitHub token (optional, never shown again)",includePullRequests: "Treat pull requests as tickets too",interval: "Sync schedule",diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 8ee98724..97f4ae65 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -386,6 +386,7 @@ export const zh: Strings = {custom: "自定义",github_issues: "GitHub 工单",jira_issues: "Jira 工单",+ s3: "对象存储",},sourceKindHints: {folder:@@ -399,6 +400,10 @@ export const zh: Strings = {"同步一个仓库的工单。**每张工单连同它的状态变更史**成为一篇文档——" +"何时开出、何时关闭、标签与负责人怎么变的,都带着日期。未配令牌时" +"每小时只有 60 次请求配额。",+ s3:+ "从 S3 桶里读文档,或者任何说同一套协议的东西(MinIO、Ceph、R2)。" ++ "**端点留空就是 AWS**,填了就是自建。每个对象成为一篇文档," ++ "日期取它最后一次修改的时刻。",api: "外部系统把 JSON 文档推送到这里,用这个来源自己的令牌认证。",custom:"按计划轮询一个你控制的 URL——你的服务返回 JSON 条目,Utopia 保持同步。",@@ -444,6 +449,12 @@ export const zh: Strings = {repoField: "仓库(owner/name)",jiraUrlField: "Jira 地址",jiraProjectField: "项目 key",+ s3BucketField: "桶名",+ s3PrefixField: "前缀(可选——不填则读整个桶)",+ s3EndpointField: "端点(用 AWS S3 就留空)",+ s3RegionField: "区域",+ s3KeyField: "Access Key ID",+ s3SecretField: "Secret Access Key",tokenField: "GitHub 令牌(可选,不再显示)",includePullRequests: "把 PR 也当工单收进来",interval: "同步计划",diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsxindex 60963a63..708d6744 100644--- a/web/src/pages/Library.tsx+++ b/web/src/pages/Library.tsx@@ -1270,6 +1270,7 @@ function SourceModal({| "api"| "github_issues"| "jira_issues"+ | "s3">("folder");const [name, setName] = useState("");const [icon, setIcon] = useState<string | null>(null);@@ -1280,6 +1281,12 @@ function SourceModal({const [repo, setRepo] = useState("");const [jiraUrl, setJiraUrl] = useState("");const [jiraProject, setJiraProject] = useState("");+ const [s3Bucket, setS3Bucket] = useState("");+ const [s3Prefix, setS3Prefix] = useState("");+ const [s3Endpoint, setS3Endpoint] = useState("");+ const [s3Region, setS3Region] = useState("");+ const [s3Key, setS3Key] = useState("");+ const [s3Secret, setS3Secret] = useState("");// PR 在 GitHub 的模型里也是工单。默认不收——问「工单」要的是工单;// 但有些仓库的决策记录实际写在 PR 描述里,所以给个开关const [includePrs, setIncludePrs] = useState(false);@@ -1293,7 +1300,8 @@ function SourceModal({kind === "rss" ||kind === "custom" ||kind === "github_issues" ||- kind === "jira_issues";+ kind === "jira_issues" ||+ kind === "s3";const create = useMutation({mutationFn: () => {@@ -1319,7 +1327,17 @@ function SourceModal({project: jiraProject.trim(),...(authHeader.trim() ? { auth_header: authHeader.trim() } : {}),}- : {};+ : kind === "s3"+ ? {+ bucket: s3Bucket.trim(),+ ...(s3Prefix.trim() ? { prefix: s3Prefix.trim() } : {}),+ // 端点留空 = 公有云 S3;填了就是自建,后端据此走 path-style+ ...(s3Endpoint.trim() ? { endpoint: s3Endpoint.trim() } : {}),+ ...(s3Region.trim() ? { region: s3Region.trim() } : {}),+ ...(s3Key.trim() ? { access_key_id: s3Key.trim() } : {}),+ ...(s3Secret.trim() ? { secret_access_key: s3Secret.trim() } : {}),+ }+ : {};return api.createSource(kbId, {kind,name: name.trim(),@@ -1341,7 +1359,9 @@ function SourceModal({? feedUrl.trim(): kind === "custom"? endpoint.trim()- : true);+ : kind === "s3"+ ? s3Bucket.trim()+ : true);// 注意用 div 而非 label:label 会把 :hover/click 转发给内部第一个可标记控件//(button 也算),导致图标网格/日程选择器悬停时第一个按钮常亮@@ -1377,6 +1397,7 @@ function SourceModal({"rss","github_issues","jira_issues",+ "s3","api","custom",] as const@@ -1520,6 +1541,63 @@ function SourceModal({)}</>)}+ {kind === "s3" && (+ <>+ {field(+ S.library.s3BucketField,+ <input+ className="input-dark w-full px-3 py-2 text-sm font-mono"+ placeholder="documents"+ value={s3Bucket}+ onChange={(e) => setS3Bucket(e.target.value)}+ />,+ )}+ {field(+ S.library.s3PrefixField,+ <input+ className="input-dark w-full px-3 py-2 text-sm font-mono"+ placeholder="reports/2026/"+ value={s3Prefix}+ onChange={(e) => setS3Prefix(e.target.value)}+ />,+ )}+ {field(+ S.library.s3EndpointField,+ <input+ className="input-dark w-full px-3 py-2 text-sm font-mono"+ placeholder="http://minio.internal:9000"+ value={s3Endpoint}+ onChange={(e) => setS3Endpoint(e.target.value)}+ />,+ )}+ {field(+ S.library.s3RegionField,+ <input+ className="input-dark w-full px-3 py-2 text-sm font-mono"+ placeholder="us-east-1"+ value={s3Region}+ onChange={(e) => setS3Region(e.target.value)}+ />,+ )}+ {field(+ S.library.s3KeyField,+ <input+ className="input-dark w-full px-3 py-2 text-sm font-mono"+ value={s3Key}+ onChange={(e) => setS3Key(e.target.value)}+ />,+ )}+ {field(+ S.library.s3SecretField,+ <input+ className="input-dark w-full px-3 py-2 text-sm font-mono"+ type="password"+ value={s3Secret}+ onChange={(e) => setS3Secret(e.target.value)}+ />,+ )}+ </>+ )}{kind === "github_issues" && (<>{field(diff --git a/web/src/pages/SourcesRail.tsx b/web/src/pages/SourcesRail.tsxindex 8ddaaacd..996c3f43 100644--- a/web/src/pages/SourcesRail.tsx+++ b/web/src/pages/SourcesRail.tsx@@ -67,6 +67,7 @@ export const KIND_ICON = {custom: Puzzle,github_issues: CircleDot,jira_issues: SquareKanban,+ s3: HardDrive,memory: Brain,upload: Upload,} as const;@@ -78,6 +79,7 @@ export const SYNCING_KINDS = new Set(["custom","github_issues","jira_issues",+ "s3",]);export const SYNC_DOT: Record<SourceView["last_sync_status"], string> = {