Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(unfurl): prevent ssrf (#5708)

Will Hutchinson50271b0e8784
diff --git a/services/unfurl_service/src/api/context.rs b/services/unfurl_service/src/api/context.rs
index f05ccb2b60..524c08568f 100644
--- a/services/unfurl_service/src/api/context.rs
+++ b/services/unfurl_service/src/api/context.rs
@@ -1,8 +1,10 @@
use axum::extract::FromRef;
use macro_env::Environment;
+use crate::http_safety::SsrfSafeHttpClient;
+
#[derive(Clone, FromRef)]
pub struct ApiContext {
pub environment: Environment,
- pub http_client: reqwest::Client,
+ pub http_client: SsrfSafeHttpClient,
}
diff --git a/services/unfurl_service/src/api/mod.rs b/services/unfurl_service/src/api/mod.rs
index 1d952734ce..0be0b41736 100644
--- a/services/unfurl_service/src/api/mod.rs
+++ b/services/unfurl_service/src/api/mod.rs
@@ -62,6 +62,7 @@ mod tests {
use crate::api::unfurl::get_unfurl::{GetUnfurlBulkBody, GetUnfurlBulkResponse};
use super::*;
+ use crate::http_safety::SsrfSafeHttpClient;
use crate::unfurl::{GetUnfurlResponse, GetUnfurlResponseList};
use ::unfurl::outbound::ReqwestUnfurlFetcher;
use axum::{
@@ -75,7 +76,7 @@ mod tests {
fn test_state() -> ApiContext {
ApiContext {
environment: Environment::Local,
- http_client: reqwest::Client::new(),
+ http_client: SsrfSafeHttpClient::new().expect("build safe test HTTP client"),
}
}
diff --git a/services/unfurl_service/src/api/proxy/mod.rs b/services/unfurl_service/src/api/proxy/mod.rs
index 4bb459835b..918a605321 100644
--- a/services/unfurl_service/src/api/proxy/mod.rs
+++ b/services/unfurl_service/src/api/proxy/mod.rs
@@ -1,16 +1,15 @@
use axum::Router;
use axum::body::Body;
use axum::extract::{Query, State};
-use axum::http::Request;
+use axum::http::{HeaderMap, HeaderName, Request};
use axum::response::Response;
use axum::routing::get;
use serde::Deserialize;
-use std::collections::HashSet;
use utoipa::{self, ToSchema};
use crate::http_safety::{
- FetchError, apply_size_limit, assert_not_internal, build_error_chain, check_content_length,
- validate_url,
+ FetchError, MAX_REDIRECTS, SsrfSafeHttpClient, apply_size_limit, assert_not_internal,
+ build_error_chain, check_content_length, redirect_target, validate_url,
};
/// 2 MB max proxied response size.
@@ -29,58 +28,22 @@ pub struct ProxyParams {
#[tracing::instrument(err(Debug), skip(http_client, request))]
pub async fn proxy_request_handler(
Query(params): Query<ProxyParams>,
- State(http_client): State<reqwest::Client>,
+ State(http_client): State<SsrfSafeHttpClient>,
request: Request<Body>,
) -> Result<Response, FetchError> {
let validated_url = validate_url(&params.url)?;
- assert_not_internal(&validated_url).await?;
-
- let excluded_headers: HashSet<&str> = [
- "connection",
- "keep-alive",
- "proxy-authenticate",
- "proxy-authorization",
- "te",
- "trailer",
- "transfer-encoding",
- "upgrade",
- "host",
- "content-length",
- ]
- .into_iter()
- .collect();
-
- let mut req_builder = http_client.get(validated_url.as_str());
- for (key, value) in request.headers().iter() {
- if !excluded_headers.contains(key.as_str()) {
- req_builder = req_builder.header(key, value);
- }
- }
-
- let response = req_builder.send().await.map_err(|e| {
- let error_chain = build_error_chain(&e);
- tracing::warn!(url = %validated_url, error = %error_chain, "upstream proxy request failed");
- if e.is_timeout() {
- FetchError::UpstreamTimeout(error_chain)
- } else if e.is_connect() {
- FetchError::UpstreamConnect(error_chain)
- } else if e.is_redirect() {
- FetchError::UpstreamRedirect(error_chain)
- } else {
- FetchError::UpstreamNetwork(error_chain)
- }
- })?;
-
- if !response.status().is_success() {
- return Err(FetchError::UpstreamStatus(response.status()));
- }
+ let forwarded_headers = forwarded_request_headers(request.headers());
+ let response = fetch_upstream(http_client.as_ref(), validated_url, &forwarded_headers).await?;
check_content_length(&response, MAX_RESPONSE_SIZE, &params.url)?;
let status = response.status();
+ let response_headers = response.headers().clone();
let mut response_builder = Response::builder().status(status);
- for (header, value) in response.headers() {
- response_builder = response_builder.header(header, value);
+ for (header, value) in response_headers.iter() {
+ if is_allowed_response_header(header) {
+ response_builder = response_builder.header(header, value);
+ }
}
let size_limited = apply_size_limit(
@@ -91,6 +54,7 @@ pub async fn proxy_request_handler(
response_builder
.header("Cross-Origin-Resource-Policy", "cross-origin")
+ .header("X-Content-Type-Options", "nosniff")
.body(Body::from_stream(size_limited))
.map_err(|e| {
tracing::error!(error=?e, "could not stream chunks");
@@ -98,6 +62,84 @@ pub async fn proxy_request_handler(
})
}
+async fn fetch_upstream(
+ http_client: &reqwest::Client,
+ mut url: url::Url,
+ headers: &HeaderMap,
+) -> Result<reqwest::Response, FetchError> {
+ let mut redirects_remaining = MAX_REDIRECTS;
+
+ loop {
+ assert_not_internal(&url).await?;
+ let response = http_client
+ .get(url.as_str())
+ .headers(headers.clone())
+ .send()
+ .await
+ .map_err(|e| {
+ let error_chain = build_error_chain(&e);
+ tracing::warn!(url = %url, error = %error_chain, "upstream proxy request failed");
+ if e.is_timeout() {
+ FetchError::UpstreamTimeout(error_chain)
+ } else if e.is_connect() {
+ FetchError::UpstreamConnect(error_chain)
+ } else if e.is_redirect() {
+ FetchError::UpstreamRedirect(error_chain)
+ } else {
+ FetchError::UpstreamNetwork(error_chain)
+ }
+ })?;
+ let status = response.status();
+
+ if status.is_redirection() {
+ if redirects_remaining == 0 {
+ return Err(FetchError::UpstreamRedirect(format!(
+ "exceeded maximum of {MAX_REDIRECTS} redirects"
+ )));
+ }
+ let next = redirect_target(&url, &response)?;
+ tracing::debug!(from = %url, to = %next, "following proxy redirect");
+ redirects_remaining -= 1;
+ url = next;
+ continue;
+ }
+
+ if !status.is_success() {
+ return Err(FetchError::UpstreamStatus(status));
+ }
+
+ return Ok(response);
+ }
+}
+
+fn forwarded_request_headers(headers: &HeaderMap) -> HeaderMap {
+ headers
+ .iter()
+ .filter(|(name, _)| is_allowed_request_header(name))
+ .map(|(name, value)| (name.clone(), value.clone()))
+ .collect()
+}
+
+fn is_allowed_request_header(header: &HeaderName) -> bool {
+ matches!(header.as_str(), "accept" | "accept-language" | "user-agent")
+}
+
+fn is_allowed_response_header(header: &HeaderName) -> bool {
+ matches!(
+ header.as_str(),
+ "cache-control"
+ | "content-encoding"
+ | "content-language"
+ | "content-type"
+ | "etag"
+ | "expires"
+ | "last-modified"
+ )
+}
+
pub fn router() -> Router<crate::api::context::ApiContext> {
Router::new().route("/", get(proxy_request_handler))
}
+
+#[cfg(test)]
+mod test;
diff --git a/services/unfurl_service/src/api/proxy/test.rs b/services/unfurl_service/src/api/proxy/test.rs
new file mode 100644
index 0000000000..e33d946051
--- /dev/null
+++ b/services/unfurl_service/src/api/proxy/test.rs
@@ -0,0 +1,66 @@
+use super::*;
+use axum::http::header::{AUTHORIZATION, COOKIE, REFERER, SET_COOKIE};
+
+fn redirect_response(location: &str) -> reqwest::Response {
+ axum::http::Response::builder()
+ .status(302)
+ .header("Location", location)
+ .body("")
+ .unwrap()
+ .into()
+}
+
+#[test]
+fn forwarded_headers_use_a_safe_allowlist() {
+ let mut incoming = HeaderMap::new();
+ incoming.insert("accept", "image/avif,image/webp,*/*".parse().unwrap());
+ incoming.insert("accept-language", "en-US".parse().unwrap());
+ incoming.insert("user-agent", "test-browser".parse().unwrap());
+ incoming.insert(AUTHORIZATION, "Bearer secret".parse().unwrap());
+ incoming.insert(COOKIE, "session=secret".parse().unwrap());
+ incoming.insert(REFERER, "https://app.macro.com/private".parse().unwrap());
+ incoming.insert("x-forwarded-for", "10.0.0.1".parse().unwrap());
+
+ let forwarded = forwarded_request_headers(&incoming);
+
+ assert_eq!(forwarded.get("accept"), incoming.get("accept"));
+ assert_eq!(
+ forwarded.get("accept-language"),
+ incoming.get("accept-language")
+ );
+ assert_eq!(forwarded.get("user-agent"), incoming.get("user-agent"));
+ assert!(!forwarded.contains_key(AUTHORIZATION));
+ assert!(!forwarded.contains_key(COOKIE));
+ assert!(!forwarded.contains_key(REFERER));
+ assert!(!forwarded.contains_key("x-forwarded-for"));
+}
+
+#[test]
+fn response_headers_exclude_credentials_and_hop_by_hop_metadata() {
+ assert!(is_allowed_response_header(&reqwest::header::CONTENT_TYPE));
+ assert!(is_allowed_response_header(&reqwest::header::CACHE_CONTROL));
+ assert!(!is_allowed_response_header(&SET_COOKIE));
+ assert!(!is_allowed_response_header(&reqwest::header::CONNECTION));
+ assert!(!is_allowed_response_header(&reqwest::header::LOCATION));
+}
+
+#[test]
+fn relative_redirects_remain_supported() {
+ let current = url::Url::parse("https://example.com/assets/icon").unwrap();
+ let response = redirect_response("../favicon.ico");
+
+ let target = redirect_target(&current, &response).unwrap();
+
+ assert_eq!(target.as_str(), "https://example.com/favicon.ico");
+}
+
+#[tokio::test]
+async fn redirect_to_metadata_endpoint_is_rejected_before_following() {
+ let current = url::Url::parse("https://example.com/start").unwrap();
+ let response = redirect_response("http://169.254.169.254/latest/meta-data/");
+ let target = redirect_target(&current, &response).unwrap();
+
+ let result = assert_not_internal(&target).await;
+
+ assert!(matches!(result, Err(FetchError::PrivateIp)));
+}
diff --git a/services/unfurl_service/src/api/unfurl/get_unfurl.rs b/services/unfurl_service/src/api/unfurl/get_unfurl.rs
index e595ba5188..5b8ccada63 100644
--- a/services/unfurl_service/src/api/unfurl/get_unfurl.rs
+++ b/services/unfurl_service/src/api/unfurl/get_unfurl.rs
@@ -2,6 +2,7 @@ use axum::extract::Json;
use axum::extract::State;
use axum::http::StatusCode;
+use crate::http_safety::SsrfSafeHttpClient;
use crate::unfurl::GetUnfurlResponseList;
use crate::unfurl::fetch_links_async;
use serde::{Deserialize, Serialize};
@@ -31,10 +32,10 @@ pub struct GetUnfurlBulkBody {
))]
#[tracing::instrument(skip(http_client, body))]
pub async fn get_bulk_unfurl_handler(
- State(http_client): State<reqwest::Client>,
+ State(http_client): State<SsrfSafeHttpClient>,
body: Json<GetUnfurlBulkBody>,
) -> (StatusCode, Json<GetUnfurlBulkResponse>) {
- let links = fetch_links_async(&http_client, &body.url_list).await;
+ let links = fetch_links_async(http_client.as_ref(), &body.url_list).await;
let response = GetUnfurlBulkResponse { responses: links };
(StatusCode::OK, Json(response))
}
diff --git a/services/unfurl_service/src/http_safety/mod.rs b/services/unfurl_service/src/http_safety/mod.rs
index 2b680a9232..cdbb0c351f 100644
--- a/services/unfurl_service/src/http_safety/mod.rs
+++ b/services/unfurl_service/src/http_safety/mod.rs
@@ -1,11 +1,46 @@
+mod resolver;
+
use axum::response::{IntoResponse, Response};
use futures::StreamExt;
use reqwest::StatusCode;
use std::error::Error;
use std::fmt;
use std::net::IpAddr;
+use std::sync::Arc;
use url::Url;
+use resolver::PrivateIpFilteringResolver;
+
+/// Maximum redirects followed by unfurl and proxy fetches.
+pub const MAX_REDIRECTS: u8 = 5;
+
+const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(8);
+const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3);
+
+/// HTTP client with redirect and DNS-rebinding SSRF protections built in.
+#[derive(Clone)]
+pub struct SsrfSafeHttpClient(reqwest::Client);
+
+impl SsrfSafeHttpClient {
+ /// Build a client with automatic redirects disabled and private DNS
+ /// connection targets filtered.
+ pub fn new() -> Result<Self, reqwest::Error> {
+ reqwest::Client::builder()
+ .timeout(REQUEST_TIMEOUT)
+ .connect_timeout(CONNECT_TIMEOUT)
+ .redirect(reqwest::redirect::Policy::none())
+ .dns_resolver(Arc::new(PrivateIpFilteringResolver))
+ .build()
+ .map(Self)
+ }
+}
+
+impl AsRef<reqwest::Client> for SsrfSafeHttpClient {
+ fn as_ref(&self) -> &reqwest::Client {
+ &self.0
+ }
+}
+
#[derive(Debug)]
pub enum FetchError {
InvalidUrl(String),
@@ -119,7 +154,10 @@ pub fn is_private_ip(ip: &IpAddr) -> bool {
if let Some(mapped_v4) = v6.to_ipv4_mapped() {
return is_private_ip(&IpAddr::V4(mapped_v4));
}
- v6.is_loopback() || v6.is_unspecified()
+ v6.is_loopback()
+ || v6.is_unspecified()
+ || v6.is_unique_local()
+ || v6.is_unicast_link_local()
}
}
}
@@ -128,7 +166,7 @@ pub async fn send_request(
http_client: &reqwest::Client,
url: &Url,
) -> Result<reqwest::Response, FetchError> {
- let response = http_client.get(url.as_str()).send().await.map_err(|e| {
+ http_client.get(url.as_str()).send().await.map_err(|e| {
let error_chain = build_error_chain(&e);
tracing::warn!(url = %url, error = %error_chain, "upstream request failed");
if e.is_timeout() {
@@ -140,13 +178,30 @@ pub async fn send_request(
} else {
FetchError::UpstreamNetwork(error_chain)
}
+ })
+}
+
+/// Resolve a redirect's `Location` header against the current URL.
+pub fn redirect_target(current: &Url, response: &reqwest::Response) -> Result<Url, FetchError> {
+ let location = response
+ .headers()
+ .get(reqwest::header::LOCATION)
+ .ok_or_else(|| {
+ FetchError::UpstreamRedirect("redirect response missing Location header".into())
+ })?
+ .to_str()
+ .map_err(|e| FetchError::UpstreamRedirect(format!("invalid Location header: {e}")))?;
+
+ let mut next = current.join(location).map_err(|e| {
+ FetchError::UpstreamRedirect(format!("could not parse redirect target {location}: {e}"))
})?;
- if !response.status().is_success() {
- return Err(FetchError::UpstreamStatus(response.status()));
+ if next.scheme() != "http" && next.scheme() != "https" {
+ return Err(FetchError::InvalidScheme);
}
+ next.set_fragment(None);
- Ok(response)
+ Ok(next)
}
pub fn content_type_of(response: &reqwest::Response) -> String {
diff --git a/services/unfurl_service/src/http_safety/resolver.rs b/services/unfurl_service/src/http_safety/resolver.rs
new file mode 100644
index 0000000000..489dd05933
--- /dev/null
+++ b/services/unfurl_service/src/http_safety/resolver.rs
@@ -0,0 +1,47 @@
+//! DNS resolver that rejects private and internal connection targets.
+
+use std::net::SocketAddr;
+
+use reqwest::dns::{Addrs, Name, Resolve, Resolving};
+
+use super::is_private_ip;
+
+pub(super) struct PrivateIpFilteringResolver;
+
+impl Resolve for PrivateIpFilteringResolver {
+ fn resolve(&self, name: Name) -> Resolving {
+ Box::pin(async move {
+ let hostname = name.as_str().to_owned();
+ let lookup = tokio::net::lookup_host(format!("{hostname}:0"))
+ .await
+ .map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)?;
+
+ let filtered: Vec<SocketAddr> = lookup
+ .filter(|address| {
+ let allowed = !is_private_ip(&address.ip());
+ if !allowed {
+ tracing::warn!(
+ host = %hostname,
+ ip = %address.ip(),
+ "dropped resolved private/internal IP from connection candidates"
+ );
+ }
+ allowed
+ })
+ .collect();
+
+ if filtered.is_empty() {
+ return Err(Box::new(std::io::Error::new(
+ std::io::ErrorKind::PermissionDenied,
+ format!(
+ "all resolved IPs for {hostname} are private/internal; refusing to connect"
+ ),
+ ))
+ as Box<dyn std::error::Error + Send + Sync>);
+ }
+
+ let addresses: Addrs = Box::new(filtered.into_iter());
+ Ok(addresses)
+ })
+ }
+}
diff --git a/services/unfurl_service/src/http_safety/test.rs b/services/unfurl_service/src/http_safety/test.rs
index c4913323b0..caff936213 100644
--- a/services/unfurl_service/src/http_safety/test.rs
+++ b/services/unfurl_service/src/http_safety/test.rs
@@ -62,6 +62,13 @@ fn test_is_private_ip_unspecified() {
assert!(is_private_ip(&"::".parse().unwrap()));
}
+#[test]
+fn test_is_private_ip_v6_unique_and_link_local() {
+ assert!(is_private_ip(&"fc00::1".parse().unwrap()));
+ assert!(is_private_ip(&"fd12:3456:789a::1".parse().unwrap()));
+ assert!(is_private_ip(&"fe80::1".parse().unwrap()));
+}
+
#[test]
fn test_is_private_ip_mapped_v4() {
assert!(is_private_ip(&"::ffff:127.0.0.1".parse().unwrap()));
@@ -78,6 +85,40 @@ fn test_is_private_ip_public() {
assert!(!is_private_ip(&"2607:f8b0:4004:800::200e".parse().unwrap()));
}
+fn redirect_response(location: Option<&str>) -> reqwest::Response {
+ let mut builder = axum::http::Response::builder().status(302);
+ if let Some(location) = location {
+ builder = builder.header("Location", location);
+ }
+ builder.body("").unwrap().into()
+}
+
+#[test]
+fn test_redirect_target_supports_absolute_and_relative_locations() {
+ let current = Url::parse("https://example.com/a/page").unwrap();
+
+ let relative = redirect_target(&current, &redirect_response(Some("../next"))).unwrap();
+ assert_eq!(relative.as_str(), "https://example.com/next");
+
+ let absolute = redirect_target(
+ &current,
+ &redirect_response(Some("https://cdn.example.com/final#fragment")),
+ )
+ .unwrap();
+ assert_eq!(absolute.as_str(), "https://cdn.example.com/final");
+}
+
+#[test]
+fn test_redirect_target_rejects_unsafe_or_missing_locations() {
+ let current = Url::parse("https://example.com/start").unwrap();
+
+ let unsafe_scheme = redirect_target(&current, &redirect_response(Some("file:///etc/passwd")));
+ assert!(matches!(unsafe_scheme, Err(FetchError::InvalidScheme)));
+
+ let missing = redirect_target(&current, &redirect_response(None));
+ assert!(matches!(missing, Err(FetchError::UpstreamRedirect(_))));
+}
+
#[tokio::test]
async fn test_assert_not_internal_blocks_loopback() {
let url = Url::parse("http://127.0.0.1/secret").unwrap();
diff --git a/services/unfurl_service/src/main.rs b/services/unfurl_service/src/main.rs
index ee620f3cd0..a8af5e1d13 100644
--- a/services/unfurl_service/src/main.rs
+++ b/services/unfurl_service/src/main.rs
@@ -20,12 +20,11 @@ async fn main() -> anyhow::Result<()> {
tracing::trace!("initialized config");
- let http_client = reqwest::Client::builder()
- .timeout(std::time::Duration::from_secs(8))
- .connect_timeout(std::time::Duration::from_secs(3))
- .redirect(reqwest::redirect::Policy::limited(5))
- .build()
- .context("failed to build http client")?;
+ // The shared client serves the legacy bulk-unfurl and proxy routes.
+ // Redirects are handled manually so every hop can be checked, while the
+ // custom resolver rejects DNS rebinding at connection time.
+ let http_client = http_safety::SsrfSafeHttpClient::new()
+ .context("failed to build shared safe http client")?;
let state = api::context::ApiContext {
environment: config.environment,
diff --git a/services/unfurl_service/src/unfurl/mod.rs b/services/unfurl_service/src/unfurl/mod.rs
index b954a8323b..c988a19c10 100644
--- a/services/unfurl_service/src/unfurl/mod.rs
+++ b/services/unfurl_service/src/unfurl/mod.rs
@@ -5,8 +5,8 @@ use std::collections::HashMap;
use url::Url;
use crate::http_safety::{
- FetchError, assert_not_internal, check_content_length, content_type_of, send_request,
- validate_url,
+ FetchError, MAX_REDIRECTS, assert_not_internal, check_content_length, content_type_of,
+ redirect_target, send_request, validate_url,
};
pub use ::unfurl::domain::favicon::append_optimistic_favico;
@@ -118,10 +118,34 @@ pub async fn extract_meta_tags_prod(
client: &reqwest::Client,
raw_url: &str,
) -> Result<HashMap<String, String>, UnfurlFetchError> {
- let url = validate_url(raw_url)?;
- assert_not_internal(&url).await?;
+ let mut url = validate_url(raw_url)?;
+ let mut redirects_remaining = MAX_REDIRECTS;
+
+ let response = loop {
+ assert_not_internal(&url).await?;
+ let response = send_request(client, &url).await?;
+ let status = response.status();
+
+ if status.is_redirection() {
+ if redirects_remaining == 0 {
+ return Err(FetchError::UpstreamRedirect(format!(
+ "exceeded maximum of {MAX_REDIRECTS} redirects"
+ ))
+ .into());
+ }
+ let next = redirect_target(&url, &response)?;
+ tracing::debug!(from = %url, to = %next, "following redirect");
+ redirects_remaining -= 1;
+ url = next;
+ continue;
+ }
+
+ if !status.is_success() {
+ return Err(FetchError::UpstreamStatus(status).into());
+ }
- let response = send_request(client, &url).await?;
+ break response;
+ };
let content_type = content_type_of(&response);
if !is_html_content_type(&content_type) {