Skip to content

World's first open-source enterprise world model.

One guarded fetch path, and it checks where it lands (#339)

WaylandYangdb4f02d76f43
diff --git a/Cargo.lock b/Cargo.lock
index e15558c0..91b454ff 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5158,6 +5158,7 @@ dependencies = [
"sqlparser",
"sqlx",
"subtle",
+ "thiserror 2.0.20",
"tokio",
"tower-http",
"tracing",
diff --git a/crates/utopia-server/Cargo.toml b/crates/utopia-server/Cargo.toml
index 306d3c31..1d16872b 100644
--- a/crates/utopia-server/Cargo.toml
+++ b/crates/utopia-server/Cargo.toml
@@ -26,6 +26,7 @@ sqlx.workspace = true
serde.workspace = true
serde_json.workspace = true
anyhow.workspace = true
+thiserror.workspace = true
uuid.workspace = true
chrono.workspace = true
argon2.workspace = true
diff --git a/crates/utopia-server/src/http_fetch.rs b/crates/utopia-server/src/http_fetch.rs
new file mode 100644
index 00000000..5d32fd85
--- /dev/null
+++ b/crates/utopia-server/src/http_fetch.rs
@@ -0,0 +1,451 @@
+//! 一条受管的抓取路径(#330)。
+//!
+//! 从前每个来源各建各的 `reqwest::Client`——六处,每处只设了超时和 UA,没有
+//! 一处问过「这个请求要连到哪里去」。于是 `http://169.254.169.254/latest/meta-data/`
+//! 会被当成一篇网页抓回来、切块、索引;重定向默认跟十跳,公网主机可以把
+//! 第二跳指进内网;响应体一次 `bytes()` 读进内存,没有上限。
+//!
+//! **谁选的这个地址,决定该有多严。**
+//!
+//! - [`Reach::Operator`]:地址是这个库的 editor 自己填的(url / webdav / jira /
+//! custom 来源)。内网是**正当目标**——自部署的内部 wiki 正是这个产品的用途,
+//! `sync_custom` 至今为回环地址专门关掉了代理。
+//! - [`Reach::Content`]:地址来自抓回来的内容(feed 里的文章链接)。这时
+//! 「一个人选的」已经退化成「一个 feed 说的」,只准公网。
+//!
+//! 两种都拿到的东西:解析后**逐个地址校验**、`resolve_to_addrs` 把地址钉死
+//! (校验完再解析一次就能被改掉,那是 DNS rebinding 的整个手法)、重定向手动
+//! 逐跳跟并复验、https 不许降级成 http、响应体有字节上限、整趟有总时限。
+//!
+//! **重定向不许升级可达范围**:operator 填的是公网主机时,第二跳就不能指进内网——
+//! 他选的是那台机器,不是那台机器的转发能力。
+//!
+//! 形状与实现要点来自 #326(@J-i-K)为 RSS 全文抓取写的那一版;这里把它抽成
+//! 所有来源共用的一条路。
+
+use std::collections::HashSet;
+use std::net::{IpAddr, SocketAddr};
+use std::time::{Duration, Instant};
+
+use futures_util::StreamExt;
+use reqwest::Url;
+
+/// 目的地是谁选的。
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Reach {
+ /// 库里的 editor 自己填的主机:内网也放行
+ Operator,
+ /// 抓回来的内容里给出的地址:只准公网
+ Content,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct Limits {
+ pub max_bytes: usize,
+ pub max_redirects: usize,
+ pub connect_timeout: Duration,
+ pub overall_timeout: Duration,
+}
+
+impl Default for Limits {
+ fn default() -> Self {
+ Self {
+ max_bytes: 16 * 1024 * 1024,
+ max_redirects: 5,
+ connect_timeout: Duration::from_secs(10),
+ overall_timeout: Duration::from_secs(30),
+ }
+ }
+}
+
+impl Limits {
+ pub fn with_overall(mut self, overall: Duration) -> Self {
+ self.overall_timeout = overall;
+ self
+ }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum FetchError {
+ #[error("invalid URL: {0}")]
+ InvalidUrl(String),
+ #[error("only http and https are fetched, got {0:?}")]
+ UnsupportedScheme(String),
+ #[error("destination {0} is not a public address")]
+ BlockedAddress(IpAddr),
+ #[error("could not resolve {0}")]
+ Dns(String),
+ #[error("redirect to {0} is not allowed from here")]
+ BlockedRedirect(String),
+ #[error("too many redirects")]
+ RedirectLimit,
+ #[error("response is larger than {0} bytes")]
+ TooLarge(usize),
+ #[error("HTTP {0}")]
+ Status(u16),
+ #[error(transparent)]
+ Transport(#[from] reqwest::Error),
+}
+
+#[derive(Debug)]
+pub struct Fetched {
+ pub final_url: Url,
+ pub mime: String,
+ pub bytes: Vec<u8>,
+}
+
+/// 一个只连得到这个主机、且地址已按 `reach` 校验过的 client。
+///
+/// 给那些自己要发好几个请求的来源用(GitHub / Jira / 自定义端点 / WebDAV 都要
+/// 带认证头、要翻页、要发 PROPFIND)——它们保留自己的请求逻辑,只是换一个
+/// 连得到的地方受管的 client。
+pub async fn client_for(
+ url: &Url,
+ reach: Reach,
+ limits: Limits,
+) -> Result<reqwest::Client, FetchError> {
+ build_client(url, reach, limits, true).await
+}
+
+/// `follow`:`get` 自己逐跳跟并复验,所以它要一个**不跟**的 client;
+/// 那些自己发请求的来源(GitHub / Jira / WebDAV)保持库默认的跟随行为。
+async fn build_client(
+ url: &Url,
+ reach: Reach,
+ limits: Limits,
+ follow: bool,
+) -> Result<reqwest::Client, FetchError> {
+ let host = url
+ .host_str()
+ .ok_or_else(|| FetchError::InvalidUrl(url.to_string()))?;
+ let addrs = resolve(host, url, reach).await?;
+ let private = addrs.iter().any(|a| !is_public_ip(a.ip()));
+ let mut builder = reqwest::Client::builder()
+ .connect_timeout(limits.connect_timeout)
+ .timeout(limits.overall_timeout)
+ .user_agent(crate::ingest_sources::UA)
+ .resolve_to_addrs(host, &addrs);
+ if !follow {
+ builder = builder.redirect(reqwest::redirect::Policy::none());
+ }
+ // 内网/回环不走系统代理:代理对这些地址只会 502(`sync_custom` 一直是这么做的)
+ if private {
+ builder = builder.no_proxy();
+ }
+ Ok(builder.build()?)
+}
+
+/// GET 一个 URL,自己跟重定向。
+pub async fn get(raw_url: &str, reach: Reach, limits: Limits) -> Result<Fetched, FetchError> {
+ let mut url = parse(raw_url)?;
+ let deadline = Instant::now() + limits.overall_timeout;
+
+ for hop in 0..=limits.max_redirects {
+ let remaining = deadline.saturating_duration_since(Instant::now());
+ if remaining.is_zero() {
+ return Err(FetchError::Status(408));
+ }
+ // **自带的重定向要关掉**:跟哪一跳由上面那个循环决定,逐跳复验;
+ // 交给 reqwest 跟就等于没人看它跟去了哪里
+ let client = build_client(&url, reach, limits.with_overall(remaining), false).await?;
+ let response = client
+ .get(url.clone())
+ .header(reqwest::header::ACCEPT_ENCODING, "identity")
+ .send()
+ .await?;
+
+ if response.status().is_redirection() {
+ if hop == limits.max_redirects {
+ return Err(FetchError::RedirectLimit);
+ }
+ let location = response
+ .headers()
+ .get(reqwest::header::LOCATION)
+ .and_then(|v| v.to_str().ok())
+ .ok_or_else(|| FetchError::BlockedRedirect("(no location)".into()))?;
+ let next = url
+ .join(location)
+ .map_err(|_| FetchError::BlockedRedirect(location.to_string()))?;
+ check_hop(&url, &next, reach).await?;
+ url = next;
+ continue;
+ }
+ if !response.status().is_success() {
+ return Err(FetchError::Status(response.status().as_u16()));
+ }
+ return read_bounded(response, url, limits.max_bytes).await;
+ }
+ Err(FetchError::RedirectLimit)
+}
+
+/// 一跳是否走得过去。
+///
+/// 除了 `reach` 本身的规则,还有一条**不许升级**:这一跳原本落在公网上,
+/// 下一跳就不能落进内网。operator 选的是那台主机,不是那台主机的转发能力。
+async fn check_hop(from: &Url, to: &Url, reach: Reach) -> Result<(), FetchError> {
+ if from.scheme() == "https" && to.scheme() == "http" {
+ return Err(FetchError::BlockedRedirect(to.to_string()));
+ }
+ let host = to
+ .host_str()
+ .ok_or_else(|| FetchError::InvalidUrl(to.to_string()))?;
+ let addrs = resolve(host, to, reach).await?;
+ let from_host = from.host_str().unwrap_or_default();
+ let from_public = resolve(from_host, from, reach)
+ .await
+ .map(|a| a.iter().all(|s| is_public_ip(s.ip())))
+ .unwrap_or(true);
+ if from_public && addrs.iter().any(|a| !is_public_ip(a.ip())) {
+ return Err(FetchError::BlockedRedirect(to.to_string()));
+ }
+ Ok(())
+}
+
+fn parse(raw: &str) -> Result<Url, FetchError> {
+ let url = Url::parse(raw.trim()).map_err(|e| FetchError::InvalidUrl(e.to_string()))?;
+ match url.scheme() {
+ "http" | "https" => Ok(url),
+ other => Err(FetchError::UnsupportedScheme(other.to_string())),
+ }
+}
+
+/// 解析主机并按 `reach` 校验每一个地址。
+///
+/// **一次解析,之后钉住**:校验完再让 client 自己解析一遍,中间那一瞬就是
+/// DNS rebinding 的下手处。端口留 0,hyper 的连接器会用 URL 上的端口。
+async fn resolve(host: &str, url: &Url, reach: Reach) -> Result<Vec<SocketAddr>, FetchError> {
+ let port = url
+ .port_or_known_default()
+ .ok_or_else(|| FetchError::InvalidUrl(url.to_string()))?;
+ let resolved: Vec<SocketAddr> = if let Ok(ip) = host.parse::<IpAddr>() {
+ vec![SocketAddr::new(ip, port)]
+ } else {
+ tokio::net::lookup_host((host, port))
+ .await
+ .map_err(|_| FetchError::Dns(host.to_string()))?
+ .collect()
+ };
+ if resolved.is_empty() {
+ return Err(FetchError::Dns(host.to_string()));
+ }
+ if reach == Reach::Content {
+ // **一个不合格就整体拒绝**,不是挑出公网的那些来连:一个主机同时解析出
+ // 公网与内网地址,本身就是要把请求引进内网的手法
+ if let Some(bad) = resolved.iter().find(|a| !is_public_ip(a.ip())) {
+ return Err(FetchError::BlockedAddress(bad.ip()));
+ }
+ }
+ let mut seen = HashSet::new();
+ Ok(resolved
+ .into_iter()
+ .filter(|a| seen.insert(a.ip()))
+ .map(|a| SocketAddr::new(a.ip(), 0))
+ .collect())
+}
+
+async fn read_bounded(
+ response: reqwest::Response,
+ final_url: Url,
+ max_bytes: usize,
+) -> Result<Fetched, FetchError> {
+ let mime = response
+ .headers()
+ .get(reqwest::header::CONTENT_TYPE)
+ .and_then(|v| v.to_str().ok())
+ .and_then(|v| v.split(';').next())
+ .map(|v| v.trim().to_ascii_lowercase())
+ .unwrap_or_else(|| "application/octet-stream".into());
+ // Content-Length 说自己超了就别开始读——但它是对方说的,不能只信它
+ if response
+ .content_length()
+ .is_some_and(|n| n as usize > max_bytes)
+ {
+ return Err(FetchError::TooLarge(max_bytes));
+ }
+ let mut bytes: Vec<u8> = Vec::new();
+ let mut stream = response.bytes_stream();
+ while let Some(chunk) = stream.next().await {
+ let chunk = chunk?;
+ if bytes.len() + chunk.len() > max_bytes {
+ return Err(FetchError::TooLarge(max_bytes));
+ }
+ bytes.extend_from_slice(&chunk);
+ }
+ Ok(Fetched {
+ final_url,
+ mime,
+ bytes,
+ })
+}
+
+/// 这个地址在公网上吗。
+///
+/// 按 RFC 6890 的特殊用途地址表拦,**只拦表里的**:`192.0.0.0/16` 整段当保留
+/// 是个常见的过度拦截(保留的只有 `192.0.0.0/24` 与 `192.0.2.0/24`),而多拦
+/// 一段就是一个用户抓不到的正当站点。
+pub fn is_public_ip(ip: IpAddr) -> bool {
+ match ip {
+ IpAddr::V4(v4) => {
+ let [a, b, c, _] = v4.octets();
+ !(v4.is_loopback() // 127.0.0.0/8
+ || v4.is_private() // 10/8, 172.16/12, 192.168/16
+ || v4.is_link_local() // 169.254/16
+ || v4.is_broadcast()
+ || v4.is_documentation() // 192.0.2/24, 198.51.100/24, 203.0.113/24
+ || v4.is_unspecified() // 0.0.0.0/8
+ || a == 0
+ || (a == 100 && (64..=127).contains(&b)) // 100.64/10 CGNAT
+ || (a == 192 && b == 0 && c == 0) // 192.0.0/24 IETF 协议分配
+ || (a == 192 && b == 88 && c == 99) // 6to4 中继任播
+ || (a == 198 && (b == 18 || b == 19)) // 198.18/15 基准测试
+ || a >= 224) // 组播与保留
+ }
+ IpAddr::V6(v6) => {
+ if let Some(v4) = v6.to_ipv4_mapped() {
+ return is_public_ip(IpAddr::V4(v4));
+ }
+ let seg = v6.segments();
+ // NAT64(64:ff9b::/96 与 64:ff9b:1::/48)裹着一个 v4 地址,按里面那个判
+ if seg[0] == 0x0064 && seg[1] == 0xff9b {
+ let v4 = std::net::Ipv4Addr::new(
+ (seg[6] >> 8) as u8,
+ seg[6] as u8,
+ (seg[7] >> 8) as u8,
+ seg[7] as u8,
+ );
+ return is_public_ip(IpAddr::V4(v4));
+ }
+ // 6to4(2002::/16)同理,v4 在第 2、3 段
+ if seg[0] == 0x2002 {
+ let v4 = std::net::Ipv4Addr::new(
+ (seg[1] >> 8) as u8,
+ seg[1] as u8,
+ (seg[2] >> 8) as u8,
+ seg[2] as u8,
+ );
+ return is_public_ip(IpAddr::V4(v4));
+ }
+ !(v6.is_loopback()
+ || v6.is_unspecified()
+ || (seg[0] & 0xff00) == 0xff00 // ff00::/8 组播
+ || (seg[0] & 0xfe00) == 0xfc00 // fc00::/7 唯一本地
+ || (seg[0] & 0xffc0) == 0xfe80 // fe80::/10 链路本地
+ || (seg[0] == 0x2001 && seg[1] == 0x0db8)) // 2001:db8::/32 文档用
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn ip(s: &str) -> IpAddr {
+ s.parse().unwrap()
+ }
+
+ #[test]
+ fn the_addresses_a_fetch_must_refuse() {
+ for blocked in [
+ "127.0.0.1",
+ "0.0.0.0",
+ "10.1.2.3",
+ "172.16.0.1",
+ "192.168.1.5",
+ "169.254.169.254", // 云上的元数据服务,这条路最常被人走
+ "100.64.0.1",
+ "192.0.0.1",
+ "198.18.0.1",
+ "224.0.0.1",
+ "::1",
+ "fc00::1",
+ "fe80::1",
+ "::ffff:127.0.0.1", // v4 映射回环
+ "64:ff9b::7f00:1", // NAT64 裹着回环
+ "2002:7f00:1::", // 6to4 裹着回环
+ ] {
+ assert!(!is_public_ip(ip(blocked)), "{blocked} 该被拦下");
+ }
+ }
+
+ #[test]
+ fn the_addresses_a_fetch_must_still_allow() {
+ for allowed in [
+ "1.1.1.1",
+ "8.8.8.8",
+ "192.0.1.1", // 192.0.0.0/16 里**不**保留的那部分
+ "192.88.98.1", // 6to4 中继只占 192.88.99.0/24
+ "2606:4700::1111",
+ "2002:0808:0808::", // 6to4 裹着 8.8.8.8
+ ] {
+ assert!(is_public_ip(ip(allowed)), "{allowed} 不该被拦");
+ }
+ }
+
+ /// 真的起一个本机服务打一遍:回环是**私有地址**,所以这套断言正好卡在
+ /// 两种 reach 的分界上——内容给的地址够不着它,操作者填的够得着。
+ #[tokio::test]
+ async fn a_loopback_host_is_reachable_only_when_a_person_typed_it() {
+ let server = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("GET"))
+ .and(wiremock::matchers::path("/page"))
+ .respond_with(
+ wiremock::ResponseTemplate::new(200)
+ .set_body_raw("<html><body>hello</body></html>", "text/html"),
+ )
+ .mount(&server)
+ .await;
+ let url = format!("{}/page", server.uri());
+
+ // feed 里的一个链接指向 127.0.0.1 —— 正是这条路要拦的东西
+ let refused = get(&url, Reach::Content, Limits::default()).await;
+ assert!(
+ matches!(refused, Err(FetchError::BlockedAddress(_))),
+ "内容给出的地址不该够得到回环,得到的是 {refused:?}"
+ );
+
+ // 同一个地址,来源是这个库的人自己填的:内部服务本来就是正当目标
+ let page = get(&url, Reach::Operator, Limits::default())
+ .await
+ .expect("操作者填的内网地址该抓得到");
+ assert_eq!(page.mime, "text/html");
+ assert!(String::from_utf8_lossy(&page.bytes).contains("hello"));
+ }
+
+ #[tokio::test]
+ async fn a_body_larger_than_the_cap_is_refused() {
+ let server = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("GET"))
+ .respond_with(wiremock::ResponseTemplate::new(200).set_body_string("x".repeat(4096)))
+ .mount(&server)
+ .await;
+ // 从前这里是一句 `resp.bytes()`:对面回多少就读多少
+ let limits = Limits {
+ max_bytes: 1024,
+ ..Limits::default()
+ };
+ let out = get(&server.uri(), Reach::Operator, limits).await;
+ assert!(matches!(out, Err(FetchError::TooLarge(1024))), "{out:?}");
+ }
+
+ #[tokio::test]
+ async fn a_redirect_chain_ends_instead_of_looping() {
+ let server = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("GET"))
+ .respond_with(wiremock::ResponseTemplate::new(302).insert_header("location", "/next"))
+ .mount(&server)
+ .await;
+ let limits = Limits {
+ max_redirects: 2,
+ ..Limits::default()
+ };
+ let out = get(&server.uri(), Reach::Operator, limits).await;
+ assert!(matches!(out, Err(FetchError::RedirectLimit)), "{out:?}");
+ }
+
+ #[test]
+ fn only_http_and_https_are_fetched() {
+ assert!(parse("file:///etc/passwd").is_err());
+ assert!(parse("gopher://example.com/").is_err());
+ assert!(parse("https://example.com/a").is_ok());
+ }
+}
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs
index 983cf5e5..5891882a 100644
--- a/crates/utopia-server/src/ingest_sources.rs
+++ b/crates/utopia-server/src/ingest_sources.rs
@@ -17,7 +17,7 @@ const MAX_NEW_PER_SYNC: usize = 200;
/// 抓取用的 User-Agent。reqwest 默认一个都不发,而维基百科明确拒绝匿名请求
/// (403),Cloudflare 前置的站点也普遍如此——URL 与 RSS 两类来源因此对一大批
/// 真实网站直接失效。自报家门也是爬虫礼节:站方能认出我们、能联系到我们。
-const UA: &str = concat!(
+pub(crate) const UA: &str = concat!(
"Utopia/",
env!("CARGO_PKG_VERSION"),
" (+https://utopia.bi; self-hosted knowledge platform)"
@@ -325,14 +325,10 @@ async fn sync_urls(state: &AppState, source: &Source) -> anyhow::Result<SyncStat
anyhow::bail!("url source is missing config.urls (a list of page URLs)");
}
- let http = reqwest::Client::builder()
- .timeout(std::time::Duration::from_secs(30))
- .user_agent(UA)
- .build()?;
let mut stats = SyncStats::default();
let mut last_err: Option<String> = None;
for url in urls.iter().take(MAX_NEW_PER_SYNC) {
- match fetch_page(&http, url).await {
+ match fetch_page(url).await {
Ok((filename, mime, bytes)) => {
// 逻辑身份 = URL 本身:页面内容变了就原地替换(历史进版本表)
let action = ingest_item(
@@ -367,26 +363,27 @@ async fn sync_urls(state: &AppState, source: &Source) -> anyhow::Result<SyncStat
Ok(stats)
}
-async fn fetch_page(
- http: &reqwest::Client,
- url: &str,
-) -> anyhow::Result<(String, String, Vec<u8>)> {
- let resp = http.get(url).send().await?;
- if !resp.status().is_success() {
- anyhow::bail!("HTTP {}", resp.status());
- }
- let mime = resp
- .headers()
- .get(reqwest::header::CONTENT_TYPE)
- .and_then(|v| v.to_str().ok())
- .unwrap_or("text/html")
- .split(';')
- .next()
- .unwrap_or("text/html")
- .to_string();
- let bytes = resp.bytes().await?.to_vec();
+/// 抓一个页面。地址是这个库的人自己填的,所以内网放行(`Reach::Operator`),
+/// 但重定向不许把请求带进内网,响应体有上限(#330)。
+async fn fetch_page(url: &str) -> anyhow::Result<(String, String, Vec<u8>)> {
+ let page = crate::http_fetch::get(
+ url,
+ crate::http_fetch::Reach::Operator,
+ crate::http_fetch::Limits::default(),
+ )
+ .await?;
+ let mime = if page.mime == "application/octet-stream" {
+ "text/html".to_string()
+ } else {
+ page.mime
+ };
+ // 身份仍然是**配置里那个 URL**(同一个页面换了地址不该变成新文档),
+ // 但落到别处这件事值得留一行
+ if page.final_url.as_str() != url {
+ tracing::debug!(from = url, to = %page.final_url, "页面跟着重定向落在别处");
+ }
let filename = filename_from_url(url, &mime);
- Ok((filename, mime, bytes))
+ Ok((filename, mime, page.bytes))
}
fn filename_from_url(url: &str, mime: &str) -> String {
@@ -425,15 +422,13 @@ async fn sync_rss(state: &AppState, source: &Source) -> anyhow::Result<SyncStats
.filter(|s| !s.trim().is_empty())
.ok_or_else(|| anyhow::anyhow!("rss source is missing config.feed_url"))?;
- let http = reqwest::Client::builder()
- .timeout(std::time::Duration::from_secs(30))
- .user_agent(UA)
- .build()?;
- let resp = http.get(feed_url).send().await?;
- if !resp.status().is_success() {
- anyhow::bail!("HTTP {} fetching feed", resp.status());
- }
- let bytes = resp.bytes().await?;
+ let feed_body = crate::http_fetch::get(
+ feed_url,
+ crate::http_fetch::Reach::Operator,
+ crate::http_fetch::Limits::default(),
+ )
+ .await?;
+ let bytes = feed_body.bytes;
let feed = feed_rs::parser::parse(&bytes[..])
.map_err(|e| anyhow::anyhow!("Failed to parse feed: {e}"))?;
@@ -523,10 +518,13 @@ async fn sync_github_issues(state: &AppState, source: &Source) -> anyhow::Result
.as_bool()
.unwrap_or(false);
- let http = reqwest::Client::builder()
- .timeout(std::time::Duration::from_secs(30))
- .user_agent(UA)
- .build()?;
+ // api.github.com 是固定的公网主机,所以按内容级的严格度收(`Reach::Content`)
+ let http = crate::http_fetch::client_for(
+ &reqwest::Url::parse("https://api.github.com/")?,
+ crate::http_fetch::Reach::Content,
+ crate::http_fetch::Limits::default(),
+ )
+ .await?;
let base = format!("https://api.github.com/repos/{repo}");
// 增量:GitHub 的 since 是"这之后更新过的"
@@ -613,10 +611,12 @@ async fn sync_jira_issues(state: &AppState, source: &Source) -> anyhow::Result<S
.map(str::trim)
.filter(|s| !s.is_empty());
- let http = reqwest::Client::builder()
- .timeout(std::time::Duration::from_secs(45))
- .user_agent(UA)
- .build()?;
+ let http = crate::http_fetch::client_for(
+ &reqwest::Url::parse(base_url).map_err(|e| anyhow::anyhow!("Invalid base_url: {e}"))?,
+ crate::http_fetch::Reach::Operator,
+ crate::http_fetch::Limits::default().with_overall(std::time::Duration::from_secs(45)),
+ )
+ .await?;
let jql = crate::jira_issues::jql(project, source.last_sync_at);
let (issues, total) = crate::jira_issues::fetch_all(&http, base_url, &jql, auth).await?;
@@ -687,20 +687,14 @@ async fn sync_custom(state: &AppState, source: &Source) -> anyhow::Result<SyncSt
url.query_pairs_mut().append_pair("since", &t.to_rfc3339());
}
- // loopback 端点不走系统代理:代理对回环地址只会 502,本机服务必须直连
- let loopback = url
- .host_str()
- .map(|h| {
- h.eq_ignore_ascii_case("localhost") || h == "127.0.0.1" || h == "::1" || h == "[::1]"
- })
- .unwrap_or(false);
- let mut builder = reqwest::Client::builder()
- .timeout(std::time::Duration::from_secs(30))
- .user_agent(UA);
- if loopback {
- builder = builder.no_proxy();
- }
- let http = builder.build()?;
+ // 端点是操作者自己填的,内网与回环都是正当目标;代理的取舍(回环走代理
+ // 只会 502)和地址校验一起收进 `client_for`(#330)
+ let http = crate::http_fetch::client_for(
+ &url,
+ crate::http_fetch::Reach::Operator,
+ crate::http_fetch::Limits::default(),
+ )
+ .await?;
let mut req = http.get(url);
if let Some(auth) = source.config["auth_header"]
.as_str()
@@ -854,9 +848,12 @@ async fn sync_webdav(state: &AppState, source: &Source) -> anyhow::Result<SyncSt
_ => None,
};
- let http = reqwest::Client::builder()
- .timeout(std::time::Duration::from_secs(60))
- .build()?;
+ let http = crate::http_fetch::client_for(
+ &reqwest::Url::parse(base).map_err(|e| anyhow::anyhow!("Invalid base_url: {e}"))?,
+ crate::http_fetch::Reach::Operator,
+ crate::http_fetch::Limits::default().with_overall(std::time::Duration::from_secs(60)),
+ )
+ .await?;
let (files, truncated) = crate::webdav::fetch(&http, base, root, auth).await?;
if truncated {
tracing::warn!(base, root, "文件数到达单次上限,其余留给下一次同步");
diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs
index 8dc05d8d..b08eefe2 100644
--- a/crates/utopia-server/src/main.rs
+++ b/crates/utopia-server/src/main.rs
@@ -9,6 +9,7 @@ mod docs_corpus;
mod error;
mod extraction;
mod github_issues;
+mod http_fetch;
mod ingest_sources;
mod jira_issues;
mod live;