World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
One guarded fetch path, and it checks where it lands (#339)
WaylandYangdb4f02d76f43
diff --git a/Cargo.lock b/Cargo.lockindex e15558c0..91b454ff 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5158,6 +5158,7 @@ dependencies = ["sqlparser","sqlx","subtle",+ "thiserror 2.0.20","tokio","tower-http","tracing",diff --git a/crates/utopia-server/Cargo.toml b/crates/utopia-server/Cargo.tomlindex 306d3c31..1d16872b 100644--- a/crates/utopia-server/Cargo.toml+++ b/crates/utopia-server/Cargo.toml@@ -26,6 +26,7 @@ sqlx.workspace = trueserde.workspace = trueserde_json.workspace = trueanyhow.workspace = true+thiserror.workspace = trueuuid.workspace = truechrono.workspace = trueargon2.workspace = truediff --git a/crates/utopia-server/src/http_fetch.rs b/crates/utopia-server/src/http_fetch.rsnew file mode 100644index 00000000..5d32fd85--- /dev/null+++ b/crates/utopia-server/src/http_fetch.rs@@ -0,0 +1,451 @@+//! 一条受管的抓取路径(#330)。+//!+//! 从前每个来源各建各的 `reqwest::Client`——六处,每处只设了超时和 UA,没有+//! 一处问过「这个请求要连到哪里去」。于是 `http://169.254.169.254/latest/meta-data/`+//! 会被当成一篇网页抓回来、切块、索引;重定向默认跟十跳,公网主机可以把+//! 第二跳指进内网;响应体一次 `bytes()` 读进内存,没有上限。+//!+//! **谁选的这个地址,决定该有多严。**+//!+//! - [`Reach::Operator`]:地址是这个库的 editor 自己填的(url / webdav / jira /+//! custom 来源)。内网是**正当目标**——自部署的内部 wiki 正是这个产品的用途,+//! `sync_custom` 至今为回环地址专门关掉了代理。+//! - [`Reach::Content`]:地址来自抓回来的内容(feed 里的文章链接)。这时+//! 「一个人选的」已经退化成「一个 feed 说的」,只准公网。+//!+//! 两种都拿到的东西:解析后**逐个地址校验**、`resolve_to_addrs` 把地址钉死+//! (校验完再解析一次就能被改掉,那是 DNS rebinding 的整个手法)、重定向手动+//! 逐跳跟并复验、https 不许降级成 http、响应体有字节上限、整趟有总时限。+//!+//! **重定向不许升级可达范围**:operator 填的是公网主机时,第二跳就不能指进内网——+//! 他选的是那台机器,不是那台机器的转发能力。+//!+//! 形状与实现要点来自 #326(@J-i-K)为 RSS 全文抓取写的那一版;这里把它抽成+//! 所有来源共用的一条路。++use std::collections::HashSet;+use std::net::{IpAddr, SocketAddr};+use std::time::{Duration, Instant};++use futures_util::StreamExt;+use reqwest::Url;++/// 目的地是谁选的。+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Reach {+ /// 库里的 editor 自己填的主机:内网也放行+ Operator,+ /// 抓回来的内容里给出的地址:只准公网+ Content,+}++#[derive(Debug, Clone, Copy)]+pub struct Limits {+ pub max_bytes: usize,+ pub max_redirects: usize,+ pub connect_timeout: Duration,+ pub overall_timeout: Duration,+}++impl Default for Limits {+ fn default() -> Self {+ Self {+ max_bytes: 16 * 1024 * 1024,+ max_redirects: 5,+ connect_timeout: Duration::from_secs(10),+ overall_timeout: Duration::from_secs(30),+ }+ }+}++impl Limits {+ pub fn with_overall(mut self, overall: Duration) -> Self {+ self.overall_timeout = overall;+ self+ }+}++#[derive(Debug, thiserror::Error)]+pub enum FetchError {+ #[error("invalid URL: {0}")]+ InvalidUrl(String),+ #[error("only http and https are fetched, got {0:?}")]+ UnsupportedScheme(String),+ #[error("destination {0} is not a public address")]+ BlockedAddress(IpAddr),+ #[error("could not resolve {0}")]+ Dns(String),+ #[error("redirect to {0} is not allowed from here")]+ BlockedRedirect(String),+ #[error("too many redirects")]+ RedirectLimit,+ #[error("response is larger than {0} bytes")]+ TooLarge(usize),+ #[error("HTTP {0}")]+ Status(u16),+ #[error(transparent)]+ Transport(#[from] reqwest::Error),+}++#[derive(Debug)]+pub struct Fetched {+ pub final_url: Url,+ pub mime: String,+ pub bytes: Vec<u8>,+}++/// 一个只连得到这个主机、且地址已按 `reach` 校验过的 client。+///+/// 给那些自己要发好几个请求的来源用(GitHub / Jira / 自定义端点 / WebDAV 都要+/// 带认证头、要翻页、要发 PROPFIND)——它们保留自己的请求逻辑,只是换一个+/// 连得到的地方受管的 client。+pub async fn client_for(+ url: &Url,+ reach: Reach,+ limits: Limits,+) -> Result<reqwest::Client, FetchError> {+ build_client(url, reach, limits, true).await+}++/// `follow`:`get` 自己逐跳跟并复验,所以它要一个**不跟**的 client;+/// 那些自己发请求的来源(GitHub / Jira / WebDAV)保持库默认的跟随行为。+async fn build_client(+ url: &Url,+ reach: Reach,+ limits: Limits,+ follow: bool,+) -> Result<reqwest::Client, FetchError> {+ let host = url+ .host_str()+ .ok_or_else(|| FetchError::InvalidUrl(url.to_string()))?;+ let addrs = resolve(host, url, reach).await?;+ let private = addrs.iter().any(|a| !is_public_ip(a.ip()));+ let mut builder = reqwest::Client::builder()+ .connect_timeout(limits.connect_timeout)+ .timeout(limits.overall_timeout)+ .user_agent(crate::ingest_sources::UA)+ .resolve_to_addrs(host, &addrs);+ if !follow {+ builder = builder.redirect(reqwest::redirect::Policy::none());+ }+ // 内网/回环不走系统代理:代理对这些地址只会 502(`sync_custom` 一直是这么做的)+ if private {+ builder = builder.no_proxy();+ }+ Ok(builder.build()?)+}++/// GET 一个 URL,自己跟重定向。+pub async fn get(raw_url: &str, reach: Reach, limits: Limits) -> Result<Fetched, FetchError> {+ let mut url = parse(raw_url)?;+ let deadline = Instant::now() + limits.overall_timeout;++ for hop in 0..=limits.max_redirects {+ let remaining = deadline.saturating_duration_since(Instant::now());+ if remaining.is_zero() {+ return Err(FetchError::Status(408));+ }+ // **自带的重定向要关掉**:跟哪一跳由上面那个循环决定,逐跳复验;+ // 交给 reqwest 跟就等于没人看它跟去了哪里+ let client = build_client(&url, reach, limits.with_overall(remaining), false).await?;+ let response = client+ .get(url.clone())+ .header(reqwest::header::ACCEPT_ENCODING, "identity")+ .send()+ .await?;++ if response.status().is_redirection() {+ if hop == limits.max_redirects {+ return Err(FetchError::RedirectLimit);+ }+ let location = response+ .headers()+ .get(reqwest::header::LOCATION)+ .and_then(|v| v.to_str().ok())+ .ok_or_else(|| FetchError::BlockedRedirect("(no location)".into()))?;+ let next = url+ .join(location)+ .map_err(|_| FetchError::BlockedRedirect(location.to_string()))?;+ check_hop(&url, &next, reach).await?;+ url = next;+ continue;+ }+ if !response.status().is_success() {+ return Err(FetchError::Status(response.status().as_u16()));+ }+ return read_bounded(response, url, limits.max_bytes).await;+ }+ Err(FetchError::RedirectLimit)+}++/// 一跳是否走得过去。+///+/// 除了 `reach` 本身的规则,还有一条**不许升级**:这一跳原本落在公网上,+/// 下一跳就不能落进内网。operator 选的是那台主机,不是那台主机的转发能力。+async fn check_hop(from: &Url, to: &Url, reach: Reach) -> Result<(), FetchError> {+ if from.scheme() == "https" && to.scheme() == "http" {+ return Err(FetchError::BlockedRedirect(to.to_string()));+ }+ let host = to+ .host_str()+ .ok_or_else(|| FetchError::InvalidUrl(to.to_string()))?;+ let addrs = resolve(host, to, reach).await?;+ let from_host = from.host_str().unwrap_or_default();+ let from_public = resolve(from_host, from, reach)+ .await+ .map(|a| a.iter().all(|s| is_public_ip(s.ip())))+ .unwrap_or(true);+ if from_public && addrs.iter().any(|a| !is_public_ip(a.ip())) {+ return Err(FetchError::BlockedRedirect(to.to_string()));+ }+ Ok(())+}++fn parse(raw: &str) -> Result<Url, FetchError> {+ let url = Url::parse(raw.trim()).map_err(|e| FetchError::InvalidUrl(e.to_string()))?;+ match url.scheme() {+ "http" | "https" => Ok(url),+ other => Err(FetchError::UnsupportedScheme(other.to_string())),+ }+}++/// 解析主机并按 `reach` 校验每一个地址。+///+/// **一次解析,之后钉住**:校验完再让 client 自己解析一遍,中间那一瞬就是+/// DNS rebinding 的下手处。端口留 0,hyper 的连接器会用 URL 上的端口。+async fn resolve(host: &str, url: &Url, reach: Reach) -> Result<Vec<SocketAddr>, FetchError> {+ let port = url+ .port_or_known_default()+ .ok_or_else(|| FetchError::InvalidUrl(url.to_string()))?;+ let resolved: Vec<SocketAddr> = if let Ok(ip) = host.parse::<IpAddr>() {+ vec![SocketAddr::new(ip, port)]+ } else {+ tokio::net::lookup_host((host, port))+ .await+ .map_err(|_| FetchError::Dns(host.to_string()))?+ .collect()+ };+ if resolved.is_empty() {+ return Err(FetchError::Dns(host.to_string()));+ }+ if reach == Reach::Content {+ // **一个不合格就整体拒绝**,不是挑出公网的那些来连:一个主机同时解析出+ // 公网与内网地址,本身就是要把请求引进内网的手法+ if let Some(bad) = resolved.iter().find(|a| !is_public_ip(a.ip())) {+ return Err(FetchError::BlockedAddress(bad.ip()));+ }+ }+ let mut seen = HashSet::new();+ Ok(resolved+ .into_iter()+ .filter(|a| seen.insert(a.ip()))+ .map(|a| SocketAddr::new(a.ip(), 0))+ .collect())+}++async fn read_bounded(+ response: reqwest::Response,+ final_url: Url,+ max_bytes: usize,+) -> Result<Fetched, FetchError> {+ let mime = response+ .headers()+ .get(reqwest::header::CONTENT_TYPE)+ .and_then(|v| v.to_str().ok())+ .and_then(|v| v.split(';').next())+ .map(|v| v.trim().to_ascii_lowercase())+ .unwrap_or_else(|| "application/octet-stream".into());+ // Content-Length 说自己超了就别开始读——但它是对方说的,不能只信它+ if response+ .content_length()+ .is_some_and(|n| n as usize > max_bytes)+ {+ return Err(FetchError::TooLarge(max_bytes));+ }+ let mut bytes: Vec<u8> = Vec::new();+ let mut stream = response.bytes_stream();+ while let Some(chunk) = stream.next().await {+ let chunk = chunk?;+ if bytes.len() + chunk.len() > max_bytes {+ return Err(FetchError::TooLarge(max_bytes));+ }+ bytes.extend_from_slice(&chunk);+ }+ Ok(Fetched {+ final_url,+ mime,+ bytes,+ })+}++/// 这个地址在公网上吗。+///+/// 按 RFC 6890 的特殊用途地址表拦,**只拦表里的**:`192.0.0.0/16` 整段当保留+/// 是个常见的过度拦截(保留的只有 `192.0.0.0/24` 与 `192.0.2.0/24`),而多拦+/// 一段就是一个用户抓不到的正当站点。+pub fn is_public_ip(ip: IpAddr) -> bool {+ match ip {+ IpAddr::V4(v4) => {+ let [a, b, c, _] = v4.octets();+ !(v4.is_loopback() // 127.0.0.0/8+ || v4.is_private() // 10/8, 172.16/12, 192.168/16+ || v4.is_link_local() // 169.254/16+ || v4.is_broadcast()+ || v4.is_documentation() // 192.0.2/24, 198.51.100/24, 203.0.113/24+ || v4.is_unspecified() // 0.0.0.0/8+ || a == 0+ || (a == 100 && (64..=127).contains(&b)) // 100.64/10 CGNAT+ || (a == 192 && b == 0 && c == 0) // 192.0.0/24 IETF 协议分配+ || (a == 192 && b == 88 && c == 99) // 6to4 中继任播+ || (a == 198 && (b == 18 || b == 19)) // 198.18/15 基准测试+ || a >= 224) // 组播与保留+ }+ IpAddr::V6(v6) => {+ if let Some(v4) = v6.to_ipv4_mapped() {+ return is_public_ip(IpAddr::V4(v4));+ }+ let seg = v6.segments();+ // NAT64(64:ff9b::/96 与 64:ff9b:1::/48)裹着一个 v4 地址,按里面那个判+ if seg[0] == 0x0064 && seg[1] == 0xff9b {+ let v4 = std::net::Ipv4Addr::new(+ (seg[6] >> 8) as u8,+ seg[6] as u8,+ (seg[7] >> 8) as u8,+ seg[7] as u8,+ );+ return is_public_ip(IpAddr::V4(v4));+ }+ // 6to4(2002::/16)同理,v4 在第 2、3 段+ if seg[0] == 0x2002 {+ let v4 = std::net::Ipv4Addr::new(+ (seg[1] >> 8) as u8,+ seg[1] as u8,+ (seg[2] >> 8) as u8,+ seg[2] as u8,+ );+ return is_public_ip(IpAddr::V4(v4));+ }+ !(v6.is_loopback()+ || v6.is_unspecified()+ || (seg[0] & 0xff00) == 0xff00 // ff00::/8 组播+ || (seg[0] & 0xfe00) == 0xfc00 // fc00::/7 唯一本地+ || (seg[0] & 0xffc0) == 0xfe80 // fe80::/10 链路本地+ || (seg[0] == 0x2001 && seg[1] == 0x0db8)) // 2001:db8::/32 文档用+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn ip(s: &str) -> IpAddr {+ s.parse().unwrap()+ }++ #[test]+ fn the_addresses_a_fetch_must_refuse() {+ for blocked in [+ "127.0.0.1",+ "0.0.0.0",+ "10.1.2.3",+ "172.16.0.1",+ "192.168.1.5",+ "169.254.169.254", // 云上的元数据服务,这条路最常被人走+ "100.64.0.1",+ "192.0.0.1",+ "198.18.0.1",+ "224.0.0.1",+ "::1",+ "fc00::1",+ "fe80::1",+ "::ffff:127.0.0.1", // v4 映射回环+ "64:ff9b::7f00:1", // NAT64 裹着回环+ "2002:7f00:1::", // 6to4 裹着回环+ ] {+ assert!(!is_public_ip(ip(blocked)), "{blocked} 该被拦下");+ }+ }++ #[test]+ fn the_addresses_a_fetch_must_still_allow() {+ for allowed in [+ "1.1.1.1",+ "8.8.8.8",+ "192.0.1.1", // 192.0.0.0/16 里**不**保留的那部分+ "192.88.98.1", // 6to4 中继只占 192.88.99.0/24+ "2606:4700::1111",+ "2002:0808:0808::", // 6to4 裹着 8.8.8.8+ ] {+ assert!(is_public_ip(ip(allowed)), "{allowed} 不该被拦");+ }+ }++ /// 真的起一个本机服务打一遍:回环是**私有地址**,所以这套断言正好卡在+ /// 两种 reach 的分界上——内容给的地址够不着它,操作者填的够得着。+ #[tokio::test]+ async fn a_loopback_host_is_reachable_only_when_a_person_typed_it() {+ let server = wiremock::MockServer::start().await;+ wiremock::Mock::given(wiremock::matchers::method("GET"))+ .and(wiremock::matchers::path("/page"))+ .respond_with(+ wiremock::ResponseTemplate::new(200)+ .set_body_raw("<html><body>hello</body></html>", "text/html"),+ )+ .mount(&server)+ .await;+ let url = format!("{}/page", server.uri());++ // feed 里的一个链接指向 127.0.0.1 —— 正是这条路要拦的东西+ let refused = get(&url, Reach::Content, Limits::default()).await;+ assert!(+ matches!(refused, Err(FetchError::BlockedAddress(_))),+ "内容给出的地址不该够得到回环,得到的是 {refused:?}"+ );++ // 同一个地址,来源是这个库的人自己填的:内部服务本来就是正当目标+ let page = get(&url, Reach::Operator, Limits::default())+ .await+ .expect("操作者填的内网地址该抓得到");+ assert_eq!(page.mime, "text/html");+ assert!(String::from_utf8_lossy(&page.bytes).contains("hello"));+ }++ #[tokio::test]+ async fn a_body_larger_than_the_cap_is_refused() {+ let server = wiremock::MockServer::start().await;+ wiremock::Mock::given(wiremock::matchers::method("GET"))+ .respond_with(wiremock::ResponseTemplate::new(200).set_body_string("x".repeat(4096)))+ .mount(&server)+ .await;+ // 从前这里是一句 `resp.bytes()`:对面回多少就读多少+ let limits = Limits {+ max_bytes: 1024,+ ..Limits::default()+ };+ let out = get(&server.uri(), Reach::Operator, limits).await;+ assert!(matches!(out, Err(FetchError::TooLarge(1024))), "{out:?}");+ }++ #[tokio::test]+ async fn a_redirect_chain_ends_instead_of_looping() {+ let server = wiremock::MockServer::start().await;+ wiremock::Mock::given(wiremock::matchers::method("GET"))+ .respond_with(wiremock::ResponseTemplate::new(302).insert_header("location", "/next"))+ .mount(&server)+ .await;+ let limits = Limits {+ max_redirects: 2,+ ..Limits::default()+ };+ let out = get(&server.uri(), Reach::Operator, limits).await;+ assert!(matches!(out, Err(FetchError::RedirectLimit)), "{out:?}");+ }++ #[test]+ fn only_http_and_https_are_fetched() {+ assert!(parse("file:///etc/passwd").is_err());+ assert!(parse("gopher://example.com/").is_err());+ assert!(parse("https://example.com/a").is_ok());+ }+}diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rsindex 983cf5e5..5891882a 100644--- a/crates/utopia-server/src/ingest_sources.rs+++ b/crates/utopia-server/src/ingest_sources.rs@@ -17,7 +17,7 @@ const MAX_NEW_PER_SYNC: usize = 200;/// 抓取用的 User-Agent。reqwest 默认一个都不发,而维基百科明确拒绝匿名请求/// (403),Cloudflare 前置的站点也普遍如此——URL 与 RSS 两类来源因此对一大批/// 真实网站直接失效。自报家门也是爬虫礼节:站方能认出我们、能联系到我们。-const UA: &str = concat!(+pub(crate) const UA: &str = concat!("Utopia/",env!("CARGO_PKG_VERSION")," (+https://utopia.bi; self-hosted knowledge platform)"@@ -325,14 +325,10 @@ async fn sync_urls(state: &AppState, source: &Source) -> anyhow::Result<SyncStatanyhow::bail!("url source is missing config.urls (a list of page URLs)");}- let http = reqwest::Client::builder()- .timeout(std::time::Duration::from_secs(30))- .user_agent(UA)- .build()?;let mut stats = SyncStats::default();let mut last_err: Option<String> = None;for url in urls.iter().take(MAX_NEW_PER_SYNC) {- match fetch_page(&http, url).await {+ match fetch_page(url).await {Ok((filename, mime, bytes)) => {// 逻辑身份 = URL 本身:页面内容变了就原地替换(历史进版本表)let action = ingest_item(@@ -367,26 +363,27 @@ async fn sync_urls(state: &AppState, source: &Source) -> anyhow::Result<SyncStatOk(stats)}-async fn fetch_page(- http: &reqwest::Client,- url: &str,-) -> anyhow::Result<(String, String, Vec<u8>)> {- let resp = http.get(url).send().await?;- if !resp.status().is_success() {- anyhow::bail!("HTTP {}", resp.status());- }- let mime = resp- .headers()- .get(reqwest::header::CONTENT_TYPE)- .and_then(|v| v.to_str().ok())- .unwrap_or("text/html")- .split(';')- .next()- .unwrap_or("text/html")- .to_string();- let bytes = resp.bytes().await?.to_vec();+/// 抓一个页面。地址是这个库的人自己填的,所以内网放行(`Reach::Operator`),+/// 但重定向不许把请求带进内网,响应体有上限(#330)。+async fn fetch_page(url: &str) -> anyhow::Result<(String, String, Vec<u8>)> {+ let page = crate::http_fetch::get(+ url,+ crate::http_fetch::Reach::Operator,+ crate::http_fetch::Limits::default(),+ )+ .await?;+ let mime = if page.mime == "application/octet-stream" {+ "text/html".to_string()+ } else {+ page.mime+ };+ // 身份仍然是**配置里那个 URL**(同一个页面换了地址不该变成新文档),+ // 但落到别处这件事值得留一行+ if page.final_url.as_str() != url {+ tracing::debug!(from = url, to = %page.final_url, "页面跟着重定向落在别处");+ }let filename = filename_from_url(url, &mime);- Ok((filename, mime, bytes))+ Ok((filename, mime, page.bytes))}fn filename_from_url(url: &str, mime: &str) -> String {@@ -425,15 +422,13 @@ async fn sync_rss(state: &AppState, source: &Source) -> anyhow::Result<SyncStats.filter(|s| !s.trim().is_empty()).ok_or_else(|| anyhow::anyhow!("rss source is missing config.feed_url"))?;- let http = reqwest::Client::builder()- .timeout(std::time::Duration::from_secs(30))- .user_agent(UA)- .build()?;- let resp = http.get(feed_url).send().await?;- if !resp.status().is_success() {- anyhow::bail!("HTTP {} fetching feed", resp.status());- }- let bytes = resp.bytes().await?;+ let feed_body = crate::http_fetch::get(+ feed_url,+ crate::http_fetch::Reach::Operator,+ crate::http_fetch::Limits::default(),+ )+ .await?;+ let bytes = feed_body.bytes;let feed = feed_rs::parser::parse(&bytes[..]).map_err(|e| anyhow::anyhow!("Failed to parse feed: {e}"))?;@@ -523,10 +518,13 @@ async fn sync_github_issues(state: &AppState, source: &Source) -> anyhow::Result.as_bool().unwrap_or(false);- let http = reqwest::Client::builder()- .timeout(std::time::Duration::from_secs(30))- .user_agent(UA)- .build()?;+ // api.github.com 是固定的公网主机,所以按内容级的严格度收(`Reach::Content`)+ let http = crate::http_fetch::client_for(+ &reqwest::Url::parse("https://api.github.com/")?,+ crate::http_fetch::Reach::Content,+ crate::http_fetch::Limits::default(),+ )+ .await?;let base = format!("https://api.github.com/repos/{repo}");// 增量:GitHub 的 since 是"这之后更新过的"@@ -613,10 +611,12 @@ async fn sync_jira_issues(state: &AppState, source: &Source) -> anyhow::Result<S.map(str::trim).filter(|s| !s.is_empty());- let http = reqwest::Client::builder()- .timeout(std::time::Duration::from_secs(45))- .user_agent(UA)- .build()?;+ let http = crate::http_fetch::client_for(+ &reqwest::Url::parse(base_url).map_err(|e| anyhow::anyhow!("Invalid base_url: {e}"))?,+ crate::http_fetch::Reach::Operator,+ crate::http_fetch::Limits::default().with_overall(std::time::Duration::from_secs(45)),+ )+ .await?;let jql = crate::jira_issues::jql(project, source.last_sync_at);let (issues, total) = crate::jira_issues::fetch_all(&http, base_url, &jql, auth).await?;@@ -687,20 +687,14 @@ async fn sync_custom(state: &AppState, source: &Source) -> anyhow::Result<SyncSturl.query_pairs_mut().append_pair("since", &t.to_rfc3339());}- // loopback 端点不走系统代理:代理对回环地址只会 502,本机服务必须直连- let loopback = url- .host_str()- .map(|h| {- h.eq_ignore_ascii_case("localhost") || h == "127.0.0.1" || h == "::1" || h == "[::1]"- })- .unwrap_or(false);- let mut builder = reqwest::Client::builder()- .timeout(std::time::Duration::from_secs(30))- .user_agent(UA);- if loopback {- builder = builder.no_proxy();- }- let http = builder.build()?;+ // 端点是操作者自己填的,内网与回环都是正当目标;代理的取舍(回环走代理+ // 只会 502)和地址校验一起收进 `client_for`(#330)+ let http = crate::http_fetch::client_for(+ &url,+ crate::http_fetch::Reach::Operator,+ crate::http_fetch::Limits::default(),+ )+ .await?;let mut req = http.get(url);if let Some(auth) = source.config["auth_header"].as_str()@@ -854,9 +848,12 @@ async fn sync_webdav(state: &AppState, source: &Source) -> anyhow::Result<SyncSt_ => None,};- let http = reqwest::Client::builder()- .timeout(std::time::Duration::from_secs(60))- .build()?;+ let http = crate::http_fetch::client_for(+ &reqwest::Url::parse(base).map_err(|e| anyhow::anyhow!("Invalid base_url: {e}"))?,+ crate::http_fetch::Reach::Operator,+ crate::http_fetch::Limits::default().with_overall(std::time::Duration::from_secs(60)),+ )+ .await?;let (files, truncated) = crate::webdav::fetch(&http, base, root, auth).await?;if truncated {tracing::warn!(base, root, "文件数到达单次上限,其余留给下一次同步");diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex 8dc05d8d..b08eefe2 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -9,6 +9,7 @@ mod docs_corpus;mod error;mod extraction;mod github_issues;+mod http_fetch;mod ingest_sources;mod jira_issues;mod live;