Skip to content

World's first open-source enterprise world model.

A bucket is a source, and MinIO speaks the same protocol (#200)

WaylandYanga14a37051621
diff --git a/Cargo.lock b/Cargo.lock
index 1710a8ee..66810570 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -707,6 +707,16 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
+[[package]]
+name = "crc-fast"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5"
+dependencies = [
+ "digest 0.10.7",
+ "spin 0.10.1",
+]
+
[[package]]
name = "crc32fast"
version = "1.5.1"
@@ -1298,7 +1308,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
dependencies = [
"futures-core",
"futures-sink",
- "spin",
+ "spin 0.9.9",
]
[[package]]
@@ -1659,6 +1669,12 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+[[package]]
+name = "humantime"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
+
[[package]]
name = "hybrid-array"
version = "0.4.14"
@@ -2002,6 +2018,15 @@ dependencies = [
"either",
]
+[[package]]
+name = "itertools"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "1.0.18"
@@ -2127,7 +2152,7 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
dependencies = [
- "spin",
+ "spin 0.9.9",
]
[[package]]
@@ -2237,7 +2262,7 @@ dependencies = [
"indexmap",
"itoa",
"log",
- "md-5",
+ "md-5 0.10.6",
"nom 8.0.0",
"rand 0.10.2",
"rangemap",
@@ -2305,6 +2330,16 @@ dependencies = [
"digest 0.10.7",
]
+[[package]]
+name = "md-5"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"
+dependencies = [
+ "cfg-if",
+ "digest 0.11.3",
+]
+
[[package]]
name = "measure_time"
version = "0.9.0"
@@ -2394,7 +2429,7 @@ dependencies = [
"httparse",
"memchr",
"mime",
- "spin",
+ "spin 0.9.9",
"version_check",
]
@@ -2410,6 +2445,18 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
+[[package]]
+name = "nix"
+version = "0.31.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "cfg_aliases",
+ "libc",
+]
+
[[package]]
name = "no_std_io2"
version = "0.9.4"
@@ -2517,6 +2564,48 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "object_store"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50"
+dependencies = [
+ "async-trait",
+ "aws-lc-rs",
+ "base64",
+ "bytes",
+ "chrono",
+ "crc-fast",
+ "form_urlencoded",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "http",
+ "http-body-util",
+ "humantime",
+ "hyper",
+ "itertools 0.15.0",
+ "md-5 0.11.0",
+ "nix",
+ "parking_lot",
+ "percent-encoding",
+ "quick-xml 0.41.0",
+ "rand 0.10.2",
+ "reqwest",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "thiserror 2.0.20",
+ "tokio",
+ "tracing",
+ "url",
+ "walkdir",
+ "wasm-bindgen-futures",
+ "web-time",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -2970,6 +3059,7 @@ checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
dependencies = [
"encoding_rs",
"memchr",
+ "serde",
]
[[package]]
@@ -3739,6 +3829,12 @@ dependencies = [
"lock_api",
]
+[[package]]
+name = "spin"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
+
[[package]]
name = "spki"
version = "0.7.3"
@@ -3874,7 +3970,7 @@ dependencies = [
"hmac",
"itoa",
"log",
- "md-5",
+ "md-5 0.10.6",
"memchr",
"once_cell",
"percent-encoding",
@@ -3915,7 +4011,7 @@ dependencies = [
"home",
"itoa",
"log",
- "md-5",
+ "md-5 0.10.6",
"memchr",
"once_cell",
"rand 0.8.8",
@@ -4128,7 +4224,7 @@ dependencies = [
"fnv",
"fs4",
"htmlescape",
- "itertools",
+ "itertools 0.14.0",
"levenshtein_automata",
"log",
"lru",
@@ -4177,7 +4273,7 @@ checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc"
dependencies = [
"downcast-rs",
"fastdivide",
- "itertools",
+ "itertools 0.14.0",
"serde",
"tantivy-bitpacker",
"tantivy-common",
@@ -4240,7 +4336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606"
dependencies = [
"futures-util",
- "itertools",
+ "itertools 0.14.0",
"tantivy-bitpacker",
"tantivy-common",
"tantivy-fst",
@@ -4299,7 +4395,7 @@ dependencies = [
"either",
"icu_provider",
"icu_segmenter",
- "itertools",
+ "itertools 0.14.0",
"memchr",
"strum",
"thiserror 2.0.20",
@@ -4831,6 +4927,7 @@ dependencies = [
"flate2",
"futures-util",
"jsonwebtoken",
+ "object_store",
"reqwest",
"serde",
"serde_json",
diff --git a/crates/utopia-server/Cargo.toml b/crates/utopia-server/Cargo.toml
index f4acec6d..a521ab5c 100644
--- a/crates/utopia-server/Cargo.toml
+++ b/crates/utopia-server/Cargo.toml
@@ -39,3 +39,4 @@ reqwest.workspace = true
feed-rs.workspace = true
async-trait = "0.1.92"
sqlparser = "0.62.0"
+object_store = { version = "0.14.1", features = ["aws"] }
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs
index 890e2964..3f8ecd1c 100644
--- a/crates/utopia-server/src/ingest_sources.rs
+++ b/crates/utopia-server/src/ingest_sources.rs
@@ -1,5 +1,6 @@
//! 来源同步任务:url(网页抓取)/ rss(订阅,pubDate → doc_time)/
-//! github_issues / jira_issues(工单,更新时刻 → doc_time,正文里带状态变更史)。
+//! github_issues / jira_issues(工单,更新时刻 → doc_time,正文里带状态变更史)/
+//! s3(对象存储,LastModified → doc_time)。
//! 全部走 sha256 去重(重复内容静默跳过),新文档进标准摄入管道(process_document)。
//! folder 是纯容器(上传入内),api 是推送型——两者无拉取语义。
@@ -53,6 +54,7 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<()
"custom" => sync_custom(state, &source).await,
"github_issues" => sync_github_issues(state, &source).await,
"jira_issues" => sync_jira_issues(state, &source).await,
+ "s3" => sync_object_storage(state, &source).await,
// folder / api 无拉取语义
_ => Ok(SyncStats::default()),
};
@@ -746,6 +748,61 @@ async fn sync_custom(state: &AppState, source: &Source) -> anyhow::Result<SyncSt
Ok(stats)
}
+/// 对象存储同步:列前缀下的对象,逐个摄入。
+///
+/// `external_key` 用 `s3://bucket/key`,跟 `file:///` 与页面 URL 同一个约定:
+/// 出处自描述。换了前缀但内容没变时,`ingest_item` 会认成「搬家」而不是新增,
+/// 不会重跑一遍抽取。
+///
+/// **`doc_time` 取 `LastModified`,而它是写入时刻不是文档自身的时间。**
+/// 一份 2019 年的合同今天传上去,时间线上会落在今天。对象存储没有更好的
+/// 来源——除非文件名或正文里带日期,而那是抽取器的活。这一条与 `url` 源
+/// 同病:`0013` 的第一条判据在这里只满足一半。
+async fn sync_object_storage(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {
+ let bucket = source.config["bucket"]
+ .as_str()
+ .map(str::trim)
+ .filter(|s| !s.is_empty())
+ .ok_or_else(|| anyhow::anyhow!("s3 source is missing config.bucket"))?;
+ let prefix = source.config["prefix"]
+ .as_str()
+ .map(str::trim)
+ .filter(|s| !s.is_empty());
+
+ let store = crate::object_storage::client(&source.config)?;
+ let (objects, truncated) = crate::object_storage::fetch(store.as_ref(), bucket, prefix).await?;
+
+ // 到顶不是错误,但必须说出来——否则「同步成功」下面藏着没进来的东西,
+ // 而那正是 0005 说的失败无声
+ if truncated {
+ tracing::warn!(
+ %bucket, prefix = prefix.unwrap_or(""),
+ "对象数到达单次上限,其余留给下一次同步"
+ );
+ }
+
+ let mut stats = SyncStats::default();
+ for obj in objects {
+ // **不猜 mime。** `utopia_ingest::parse` 先看魔数、再看扩展名,
+ // 注释写着「扩展名可能撒谎」——在这里按文件名猜一个,只是多一个
+ // 会撒谎的来源,而且要为此多一个依赖。`octet-stream` 是诚实的:
+ // 我们拿到的就是一串字节,没看过里面是什么。
+ let action = ingest_item(
+ state,
+ source.kb_id,
+ source.id,
+ &obj.external_key,
+ &obj.filename,
+ "application/octet-stream",
+ &obj.bytes,
+ obj.last_modified,
+ )
+ .await?;
+ stats.absorb(action);
+ }
+ Ok(stats)
+}
+
/// 标题无扩展名时按 mime 补一个,接入解析矩阵的分派。
fn ensure_extension(title: &str, mime: &str) -> String {
let has_ext = title
diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs
index 34217e0b..01613d88 100644
--- a/crates/utopia-server/src/main.rs
+++ b/crates/utopia-server/src/main.rs
@@ -14,6 +14,7 @@ mod jira_issues;
mod live;
mod llm_util;
mod mappings;
+mod object_storage;
mod ontology_index;
mod ontology_packs;
mod owl_import;
diff --git a/crates/utopia-server/src/object_storage.rs b/crates/utopia-server/src/object_storage.rs
new file mode 100644
index 00000000..a4f381f0
--- /dev/null
+++ b/crates/utopia-server/src/object_storage.rs
@@ -0,0 +1,275 @@
+//! 对象存储来源:S3 与一切说同一套协议的东西(MinIO、Ceph RGW、阿里 OSS 的
+//! S3 兼容端点、Cloudflare R2…)。
+//!
+//! 按 [0013](../../docs/decisions/0013-a-source-should-hand-over-its-history.md)
+//! 的四条判据看,它跟工单系统不是一类:
+//!
+//! | 判据 | 对象存储 |
+//! |---|---|
+//! | 真实时间戳 | `LastModified`,但它是**写入时刻**不是文档自身的时间 |
+//! | 会不会自我推翻 | 同 key 覆写就是一次改口,但要靠两次同步之间的差异看出来 |
+//! | 稳定身份 | `bucket/key`,最干净的一条 |
+//! | 企业知识住不住在那儿 | **最强的一条**——文档堆、归档、数据湖落地区都在这儿 |
+//!
+//! 所以它的价值在第四条:**够得着**。第二条只有部分——工单系统能一次交出
+//! 整段变更史,而对象存储要么开了版本控制(本版不读,见下),要么只能靠
+//! 一次次同步慢慢攒。这跟 `url` / `rss` 是同一档,不比它们低。
+//!
+//! **版本历史本版不做。** `ListObjectVersions` 能一次拿到 0013 想要的那段历史,
+//! 但多数桶没开版本控制,开了的桶里也大多是写一次不再动的对象——把每个版本
+//! 都摄进来,代价与收益不成比例。等有人真的需要(一份季度更新的制度文件,
+//! 想看它改了什么)再按开关加。
+//!
+//! **为什么是 `object_store` 而不是 `aws-sdk-s3` / `rust-s3`**:实测传递依赖
+//! `+7` 对 `+43` / `+42`——它复用了树里已有的 `reqwest`、`ring`、`quick-xml`。
+//! 而且同一套 API 还覆盖 Azure Blob 与 GCS,路线图上「更多数据源」与
+//! 「数据湖仓」两条都在这条线上,换后端只是换一个 builder。
+
+use anyhow::Context as _;
+use chrono::{DateTime, Utc};
+use futures_util::StreamExt as _;
+use object_store::aws::AmazonS3Builder;
+// `get` 住在 `ObjectStoreExt` 上,不在 `ObjectStore` 本体——后者只有
+// `get_opts`。少这一行的报错是「&dyn ObjectStore 没有 get 方法」,
+// 而 trait 名字里看不出来。
+use object_store::{path::Path as StorePath, ObjectStore, ObjectStoreExt as _};
+
+/// 一次同步最多摄入多少个对象。
+///
+/// **不是性能考虑,是「别把一个桶整个吸进来」。** 桶里放上百万个对象是常态,
+/// 而摄入是不可逆的:每个对象要抽取、要向量化、要进图。配错一个前缀就能
+/// 烧掉一整天的额度,而且清理起来比配置麻烦得多。
+///
+/// 到顶时报进同步统计,让人看得见「还有」,而不是悄悄截断。
+const MAX_OBJECTS_PER_SYNC: usize = 2_000;
+
+/// 单个对象的大小上限。跟上传路径同一个数量级——超过这个尺寸的多半不是
+/// 文档而是数据文件(备份、镜像、视频),抽取器拿它没有用。
+const MAX_OBJECT_BYTES: u64 = 32 * 1024 * 1024;
+
+/// 一个待摄入的对象。
+pub struct RemoteObject {
+ /// `s3://bucket/key`——`ingest_item` 的 external_key 约定是 URI 形态
+ pub external_key: String,
+ /// 落进文档的文件名,取 key 的最后一段
+ pub filename: String,
+ pub bytes: Vec<u8>,
+ pub last_modified: Option<DateTime<Utc>>,
+}
+
+/// 从来源配置建一个客户端。
+///
+/// **`endpoint` 在则走自建**(MinIO、Ceph、R2),此时强制 path-style:
+/// 虚拟主机式寻址要求 `bucket.host` 这样的域名解析得开,而自建部署通常
+/// 只有一个裸 IP 或一个主机名。不强制的话症状是「桶不存在」,
+/// 而桶明明就在那儿。
+///
+/// 允许 http 也是为它:内网 MinIO 常常不挂 TLS。**这是配置说了算的降级**,
+/// 不是默认——公有云那条路(不给 endpoint)仍然只走 https。
+pub fn client(config: &serde_json::Value) -> anyhow::Result<Box<dyn ObjectStore>> {
+ let s = |k: &str| {
+ config[k]
+ .as_str()
+ .map(str::trim)
+ .filter(|v| !v.is_empty())
+ .map(str::to_string)
+ };
+ let bucket = s("bucket")
+ .ok_or_else(|| anyhow::anyhow!("object storage source is missing config.bucket"))?;
+
+ let mut b = AmazonS3Builder::new().with_bucket_name(&bucket);
+ if let Some(region) = s("region") {
+ b = b.with_region(region);
+ }
+ if let (Some(key), Some(secret)) = (s("access_key_id"), s("secret_access_key")) {
+ b = b.with_access_key_id(key).with_secret_access_key(secret);
+ }
+ if let Some(endpoint) = s("endpoint") {
+ let insecure = endpoint.starts_with("http://");
+ b = b
+ .with_endpoint(endpoint)
+ .with_virtual_hosted_style_request(false)
+ .with_allow_http(insecure);
+ }
+ Ok(Box::new(b.build().context("object storage client")?))
+}
+
+/// 列出前缀下的对象并逐个取回。
+///
+/// **零字节的一律跳过,判据是大小不是名字。** 直觉的写法是看 key 以不以 `/`
+/// 结尾——控制台和 `aws s3 sync` 就是这样造「文件夹」的。但 `object_store`
+/// 的 `Path` 会**把尾斜杠规范化掉**,`docs/` 到手里就成了 `docs`,那个判断
+/// 永远不成立;接着它去 GET `docs`,而桶里真实的 key 是 `docs/`,
+/// 于是 404 把整次同步带走。实测在 MinIO 上就是这样炸的。
+///
+/// 按大小判还更宽:零字节的对象无论叫什么都不是文档,`ingest_item` 拿到
+/// 空字节也只会返回 `Unchanged`。
+///
+/// 格式不在这里判。`utopia_ingest` 的抽取按扩展名加 mime 分派,认不出的
+/// 一律按文本解码——在这里再维护一张白名单,就是同一件事写两遍,
+/// 而两份清单迟早分叉。
+pub async fn fetch(
+ store: &dyn ObjectStore,
+ bucket: &str,
+ prefix: Option<&str>,
+) -> anyhow::Result<(Vec<RemoteObject>, bool)> {
+ let p = prefix.map(StorePath::from);
+ let mut listing = store.list(p.as_ref());
+ let mut out = Vec::new();
+ let mut truncated = false;
+ let mut unreadable = 0usize;
+
+ while let Some(meta) = listing.next().await {
+ let meta = meta.context("listing objects")?;
+ let key = meta.location.as_ref().to_string();
+
+ if meta.size == 0 {
+ continue;
+ }
+ if meta.size > MAX_OBJECT_BYTES {
+ tracing::info!(%key, size = meta.size, "对象太大,跳过");
+ continue;
+ }
+ if out.len() >= MAX_OBJECTS_PER_SYNC {
+ truncated = true;
+ break;
+ }
+
+ // **一个对象取不回来不该带走整次同步。** 列表与取回之间隔着时间,
+ // 中途被删、被换权限都是正常的;桶越大越常见。记下来在收尾时一并报,
+ // 而不是让第 900 个对象的一次 404 把前 899 个的成果一起丢掉。
+ let got = async {
+ let r = store.get(&meta.location).await?;
+ r.bytes().await
+ }
+ .await;
+ let bytes = match got {
+ Ok(b) => b,
+ Err(e) => {
+ unreadable += 1;
+ tracing::warn!(%key, error = %e, "对象取不回来,跳过");
+ continue;
+ }
+ };
+
+ out.push(RemoteObject {
+ external_key: format!("s3://{bucket}/{key}"),
+ filename: key.rsplit('/').next().unwrap_or(&key).to_string(),
+ bytes: bytes.to_vec(),
+ last_modified: Some(meta.last_modified),
+ });
+ }
+ if unreadable > 0 {
+ tracing::warn!(unreadable, bucket, "有对象没能取回");
+ }
+ Ok((out, truncated))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// 自建端点必须走 path-style,否则请求会打到 `bucket.host` 上,
+ /// 而自建部署一般没有那条 DNS 记录。症状是「桶不存在」而桶就在那儿。
+ #[test]
+ fn a_self_hosted_endpoint_builds() {
+ let cfg = serde_json::json!({
+ "bucket": "docs",
+ "endpoint": "http://127.0.0.1:9000",
+ "region": "us-east-1",
+ "access_key_id": "minioadmin",
+ "secret_access_key": "minioadmin",
+ });
+ assert!(client(&cfg).is_ok());
+ }
+
+ /// 不给 bucket 要报得出是缺哪一项——照 jira / github 两个来源的措辞。
+ #[test]
+ fn a_missing_bucket_says_so() {
+ let e = client(&serde_json::json!({ "region": "us-east-1" })).unwrap_err();
+ assert!(e.to_string().contains("config.bucket"), "{e}");
+ }
+
+ /// 公有云那条路:不给 endpoint 就不该被强制成 path-style,
+ /// 也不该允许明文 http。
+ #[test]
+ fn a_cloud_bucket_needs_no_endpoint() {
+ let cfg = serde_json::json!({
+ "bucket": "docs",
+ "region": "ap-southeast-1",
+ "access_key_id": "AKIA...",
+ "secret_access_key": "...",
+ });
+ assert!(client(&cfg).is_ok());
+ }
+
+ /// 真连一个 S3 兼容端点,只测**读**这一侧:列出来、取回内容、身份对不对。
+ ///
+ /// **没有 `UTOPIA_S3_TEST_ENDPOINT` 时跳过而不是失败**——跟连库的测试
+ /// 同一个约定(见 CONTRIBUTING)。上面那三条只证明 builder 拼得出来,
+ /// 而这条要守的是**线上协议真的说得通**:path-style 有没有生效、
+ /// 明文 http 放没放行、`s3://` 身份拼得对不对、`LastModified` 回不回来。
+ /// 没有一条是构造 client 时能发现的。
+ ///
+ /// **数据由外部布置,测试不写只读。** 第一版让测试自己 `put`,结果被
+ /// `object_store` 的 `Path` 咬了一口:它会把 `docs/` 规范化成 `docs`,
+ /// 于是「目录占位」写成了一个名叫 `docs` 的普通对象,而 MinIO 不允许
+ /// 对象 `docs` 与前缀 `docs/` 并存——整个前缀被顶掉,列表回来是空的。
+ /// 真实的占位对象只能由别的客户端造,那就让它由别的客户端造。
+ ///
+ /// 跑法(先起 MinIO,再用 curl 布数据;curl 自带 SigV4):
+ /// ```text
+ /// MINIO_ROOT_USER=u MINIO_ROOT_PASSWORD=p123456 \
+ /// minio server ./data --address 127.0.0.1:19000 &
+ /// S3="curl -s --aws-sigv4 aws:amz:us-east-1:s3 -u u:p123456"
+ /// B=http://127.0.0.1:19000/utopia-conn-test
+ /// $S3 -X PUT $B # 建桶
+ /// $S3 -X PUT --data-raw hello "$B/docs/a.txt"
+ /// $S3 -X PUT --data-raw '# world' "$B/docs/b.md"
+ /// $S3 -X PUT --data-raw '' "$B/docs/" # 目录占位
+ /// UTOPIA_S3_TEST_ENDPOINT=http://127.0.0.1:19000 \
+ /// UTOPIA_S3_TEST_KEY=u UTOPIA_S3_TEST_SECRET=p123456 \
+ /// cargo test -p utopia-server object_storage
+ /// ```
+ #[tokio::test]
+ async fn it_reads_from_a_real_s3_endpoint() -> anyhow::Result<()> {
+ let Ok(endpoint) = std::env::var("UTOPIA_S3_TEST_ENDPOINT") else {
+ eprintln!("跳过:未设 UTOPIA_S3_TEST_ENDPOINT");
+ return Ok(());
+ };
+ let bucket = "utopia-conn-test";
+ let cfg = serde_json::json!({
+ "bucket": bucket,
+ "endpoint": endpoint,
+ "region": "us-east-1",
+ "access_key_id": std::env::var("UTOPIA_S3_TEST_KEY").unwrap_or("minioadmin".into()),
+ "secret_access_key": std::env::var("UTOPIA_S3_TEST_SECRET").unwrap_or("minioadmin".into()),
+ });
+ let store = client(&cfg)?;
+
+ let (objs, truncated) = fetch(store.as_ref(), bucket, Some("docs")).await?;
+ assert!(!truncated, "三个对象不该触发上限");
+
+ let keys: Vec<&str> = objs.iter().map(|o| o.external_key.as_str()).collect();
+ for want in [
+ "s3://utopia-conn-test/docs/a.txt",
+ "s3://utopia-conn-test/docs/b.md",
+ ] {
+ assert!(keys.contains(&want), "少了 {want}:{keys:?}");
+ }
+
+ // **这条守的是一个真炸过的 bug。** 桶里有一个 `docs/` 目录占位对象,
+ // 而 `object_store` 把尾斜杠规范化掉了,于是它以 `docs` 的身份进到
+ // 循环里,GET 回来 404,整次同步失败。三条构造 client 的单元测试
+ // 一条都发现不了——只有真连一个装过「文件夹」的桶才看得见。
+ assert_eq!(objs.len(), 2, "目录占位对象没被滤掉:{keys:?}");
+
+ let a = objs.iter().find(|o| o.filename == "a.txt").expect("a.txt");
+ assert_eq!(a.bytes, b"hello", "取回的内容不对");
+ assert!(
+ a.last_modified.is_some(),
+ "LastModified 是 doc_time 的唯一来源"
+ );
+ Ok(())
+ }
+}
diff --git a/web/src/api.ts b/web/src/api.ts
index 32d364e7..4be615e5 100644
--- a/web/src/api.ts
+++ b/web/src/api.ts
@@ -169,6 +169,7 @@ export interface SourceView {
| "custom"
| "github_issues"
| "jira_issues"
+ | "s3"
| "memory"
| "upload";
name: string;
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index a133c070..45c260c9 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -418,6 +418,7 @@ export const en = {
custom: "Custom",
github_issues: "GitHub issues",
jira_issues: "Jira issues",
+ s3: "Object storage",
},
sourceKindHints: {
folder:
@@ -432,6 +433,10 @@ export const en = {
"Syncs a repository's issues. **Each ticket, together with its state history**, becomes " +
"one document — when it was opened, closed, relabelled, reassigned, all dated. " +
"Without a token GitHub allows only 60 requests an hour.",
+ s3:
+ "Reads documents out of an S3 bucket, or anything speaking the same protocol " +
+ "(MinIO, Ceph, R2). **Leave the endpoint empty for AWS**; fill it in for a " +
+ "self-hosted one. Each object becomes a document dated by its last modification.",
api: "External systems push JSON documents here, authenticated with this source's own token.",
custom:
"Polls a URL you control on a schedule — your service returns JSON items and Utopia keeps them in sync.",
@@ -478,6 +483,12 @@ export const en = {
repoField: "Repository (owner/name)",
jiraUrlField: "Jira site URL",
jiraProjectField: "Project key",
+ s3BucketField: "Bucket",
+ s3PrefixField: "Prefix (optional — without one the whole bucket is read)",
+ s3EndpointField: "Endpoint (leave empty for AWS S3)",
+ s3RegionField: "Region",
+ s3KeyField: "Access key ID",
+ s3SecretField: "Secret access key",
tokenField: "GitHub token (optional, never shown again)",
includePullRequests: "Treat pull requests as tickets too",
interval: "Sync schedule",
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 8ee98724..97f4ae65 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -386,6 +386,7 @@ export const zh: Strings = {
custom: "自定义",
github_issues: "GitHub 工单",
jira_issues: "Jira 工单",
+ s3: "对象存储",
},
sourceKindHints: {
folder:
@@ -399,6 +400,10 @@ export const zh: Strings = {
"同步一个仓库的工单。**每张工单连同它的状态变更史**成为一篇文档——" +
"何时开出、何时关闭、标签与负责人怎么变的,都带着日期。未配令牌时" +
"每小时只有 60 次请求配额。",
+ s3:
+ "从 S3 桶里读文档,或者任何说同一套协议的东西(MinIO、Ceph、R2)。" +
+ "**端点留空就是 AWS**,填了就是自建。每个对象成为一篇文档," +
+ "日期取它最后一次修改的时刻。",
api: "外部系统把 JSON 文档推送到这里,用这个来源自己的令牌认证。",
custom:
"按计划轮询一个你控制的 URL——你的服务返回 JSON 条目,Utopia 保持同步。",
@@ -444,6 +449,12 @@ export const zh: Strings = {
repoField: "仓库(owner/name)",
jiraUrlField: "Jira 地址",
jiraProjectField: "项目 key",
+ s3BucketField: "桶名",
+ s3PrefixField: "前缀(可选——不填则读整个桶)",
+ s3EndpointField: "端点(用 AWS S3 就留空)",
+ s3RegionField: "区域",
+ s3KeyField: "Access Key ID",
+ s3SecretField: "Secret Access Key",
tokenField: "GitHub 令牌(可选,不再显示)",
includePullRequests: "把 PR 也当工单收进来",
interval: "同步计划",
diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsx
index 60963a63..708d6744 100644
--- a/web/src/pages/Library.tsx
+++ b/web/src/pages/Library.tsx
@@ -1270,6 +1270,7 @@ function SourceModal({
| "api"
| "github_issues"
| "jira_issues"
+ | "s3"
>("folder");
const [name, setName] = useState("");
const [icon, setIcon] = useState<string | null>(null);
@@ -1280,6 +1281,12 @@ function SourceModal({
const [repo, setRepo] = useState("");
const [jiraUrl, setJiraUrl] = useState("");
const [jiraProject, setJiraProject] = useState("");
+ const [s3Bucket, setS3Bucket] = useState("");
+ const [s3Prefix, setS3Prefix] = useState("");
+ const [s3Endpoint, setS3Endpoint] = useState("");
+ const [s3Region, setS3Region] = useState("");
+ const [s3Key, setS3Key] = useState("");
+ const [s3Secret, setS3Secret] = useState("");
// PR 在 GitHub 的模型里也是工单。默认不收——问「工单」要的是工单;
// 但有些仓库的决策记录实际写在 PR 描述里,所以给个开关
const [includePrs, setIncludePrs] = useState(false);
@@ -1293,7 +1300,8 @@ function SourceModal({
kind === "rss" ||
kind === "custom" ||
kind === "github_issues" ||
- kind === "jira_issues";
+ kind === "jira_issues" ||
+ kind === "s3";
const create = useMutation({
mutationFn: () => {
@@ -1319,7 +1327,17 @@ function SourceModal({
project: jiraProject.trim(),
...(authHeader.trim() ? { auth_header: authHeader.trim() } : {}),
}
- : {};
+ : kind === "s3"
+ ? {
+ bucket: s3Bucket.trim(),
+ ...(s3Prefix.trim() ? { prefix: s3Prefix.trim() } : {}),
+ // 端点留空 = 公有云 S3;填了就是自建,后端据此走 path-style
+ ...(s3Endpoint.trim() ? { endpoint: s3Endpoint.trim() } : {}),
+ ...(s3Region.trim() ? { region: s3Region.trim() } : {}),
+ ...(s3Key.trim() ? { access_key_id: s3Key.trim() } : {}),
+ ...(s3Secret.trim() ? { secret_access_key: s3Secret.trim() } : {}),
+ }
+ : {};
return api.createSource(kbId, {
kind,
name: name.trim(),
@@ -1341,7 +1359,9 @@ function SourceModal({
? feedUrl.trim()
: kind === "custom"
? endpoint.trim()
- : true);
+ : kind === "s3"
+ ? s3Bucket.trim()
+ : true);
// 注意用 div 而非 label:label 会把 :hover/click 转发给内部第一个可标记控件
//(button 也算),导致图标网格/日程选择器悬停时第一个按钮常亮
@@ -1377,6 +1397,7 @@ function SourceModal({
"rss",
"github_issues",
"jira_issues",
+ "s3",
"api",
"custom",
] as const
@@ -1520,6 +1541,63 @@ function SourceModal({
)}
</>
)}
+ {kind === "s3" && (
+ <>
+ {field(
+ S.library.s3BucketField,
+ <input
+ className="input-dark w-full px-3 py-2 text-sm font-mono"
+ placeholder="documents"
+ value={s3Bucket}
+ onChange={(e) => setS3Bucket(e.target.value)}
+ />,
+ )}
+ {field(
+ S.library.s3PrefixField,
+ <input
+ className="input-dark w-full px-3 py-2 text-sm font-mono"
+ placeholder="reports/2026/"
+ value={s3Prefix}
+ onChange={(e) => setS3Prefix(e.target.value)}
+ />,
+ )}
+ {field(
+ S.library.s3EndpointField,
+ <input
+ className="input-dark w-full px-3 py-2 text-sm font-mono"
+ placeholder="http://minio.internal:9000"
+ value={s3Endpoint}
+ onChange={(e) => setS3Endpoint(e.target.value)}
+ />,
+ )}
+ {field(
+ S.library.s3RegionField,
+ <input
+ className="input-dark w-full px-3 py-2 text-sm font-mono"
+ placeholder="us-east-1"
+ value={s3Region}
+ onChange={(e) => setS3Region(e.target.value)}
+ />,
+ )}
+ {field(
+ S.library.s3KeyField,
+ <input
+ className="input-dark w-full px-3 py-2 text-sm font-mono"
+ value={s3Key}
+ onChange={(e) => setS3Key(e.target.value)}
+ />,
+ )}
+ {field(
+ S.library.s3SecretField,
+ <input
+ className="input-dark w-full px-3 py-2 text-sm font-mono"
+ type="password"
+ value={s3Secret}
+ onChange={(e) => setS3Secret(e.target.value)}
+ />,
+ )}
+ </>
+ )}
{kind === "github_issues" && (
<>
{field(
diff --git a/web/src/pages/SourcesRail.tsx b/web/src/pages/SourcesRail.tsx
index 8ddaaacd..996c3f43 100644
--- a/web/src/pages/SourcesRail.tsx
+++ b/web/src/pages/SourcesRail.tsx
@@ -67,6 +67,7 @@ export const KIND_ICON = {
custom: Puzzle,
github_issues: CircleDot,
jira_issues: SquareKanban,
+ s3: HardDrive,
memory: Brain,
upload: Upload,
} as const;
@@ -78,6 +79,7 @@ export const SYNCING_KINDS = new Set([
"custom",
"github_issues",
"jira_issues",
+ "s3",
]);
export const SYNC_DOT: Record<SourceView["last_sync_status"], string> = {