Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(unfurl): prevent ssrf (#5708)
Will Hutchinson50271b0e8784
diff --git a/services/unfurl_service/src/api/context.rs b/services/unfurl_service/src/api/context.rsindex f05ccb2b60..524c08568f 100644--- a/services/unfurl_service/src/api/context.rs+++ b/services/unfurl_service/src/api/context.rs@@ -1,8 +1,10 @@use axum::extract::FromRef;use macro_env::Environment;+use crate::http_safety::SsrfSafeHttpClient;+#[derive(Clone, FromRef)]pub struct ApiContext {pub environment: Environment,- pub http_client: reqwest::Client,+ pub http_client: SsrfSafeHttpClient,}diff --git a/services/unfurl_service/src/api/mod.rs b/services/unfurl_service/src/api/mod.rsindex 1d952734ce..0be0b41736 100644--- a/services/unfurl_service/src/api/mod.rs+++ b/services/unfurl_service/src/api/mod.rs@@ -62,6 +62,7 @@ mod tests {use crate::api::unfurl::get_unfurl::{GetUnfurlBulkBody, GetUnfurlBulkResponse};use super::*;+ use crate::http_safety::SsrfSafeHttpClient;use crate::unfurl::{GetUnfurlResponse, GetUnfurlResponseList};use ::unfurl::outbound::ReqwestUnfurlFetcher;use axum::{@@ -75,7 +76,7 @@ mod tests {fn test_state() -> ApiContext {ApiContext {environment: Environment::Local,- http_client: reqwest::Client::new(),+ http_client: SsrfSafeHttpClient::new().expect("build safe test HTTP client"),}}diff --git a/services/unfurl_service/src/api/proxy/mod.rs b/services/unfurl_service/src/api/proxy/mod.rsindex 4bb459835b..918a605321 100644--- a/services/unfurl_service/src/api/proxy/mod.rs+++ b/services/unfurl_service/src/api/proxy/mod.rs@@ -1,16 +1,15 @@use axum::Router;use axum::body::Body;use axum::extract::{Query, State};-use axum::http::Request;+use axum::http::{HeaderMap, HeaderName, Request};use axum::response::Response;use axum::routing::get;use serde::Deserialize;-use std::collections::HashSet;use utoipa::{self, ToSchema};use crate::http_safety::{- FetchError, apply_size_limit, assert_not_internal, build_error_chain, check_content_length,- validate_url,+ FetchError, MAX_REDIRECTS, SsrfSafeHttpClient, apply_size_limit, assert_not_internal,+ build_error_chain, check_content_length, redirect_target, validate_url,};/// 2 MB max proxied response size.@@ -29,58 +28,22 @@ pub struct ProxyParams {#[tracing::instrument(err(Debug), skip(http_client, request))]pub async fn proxy_request_handler(Query(params): Query<ProxyParams>,- State(http_client): State<reqwest::Client>,+ State(http_client): State<SsrfSafeHttpClient>,request: Request<Body>,) -> Result<Response, FetchError> {let validated_url = validate_url(¶ms.url)?;- assert_not_internal(&validated_url).await?;-- let excluded_headers: HashSet<&str> = [- "connection",- "keep-alive",- "proxy-authenticate",- "proxy-authorization",- "te",- "trailer",- "transfer-encoding",- "upgrade",- "host",- "content-length",- ]- .into_iter()- .collect();-- let mut req_builder = http_client.get(validated_url.as_str());- for (key, value) in request.headers().iter() {- if !excluded_headers.contains(key.as_str()) {- req_builder = req_builder.header(key, value);- }- }-- let response = req_builder.send().await.map_err(|e| {- let error_chain = build_error_chain(&e);- tracing::warn!(url = %validated_url, error = %error_chain, "upstream proxy request failed");- if e.is_timeout() {- FetchError::UpstreamTimeout(error_chain)- } else if e.is_connect() {- FetchError::UpstreamConnect(error_chain)- } else if e.is_redirect() {- FetchError::UpstreamRedirect(error_chain)- } else {- FetchError::UpstreamNetwork(error_chain)- }- })?;-- if !response.status().is_success() {- return Err(FetchError::UpstreamStatus(response.status()));- }+ let forwarded_headers = forwarded_request_headers(request.headers());+ let response = fetch_upstream(http_client.as_ref(), validated_url, &forwarded_headers).await?;check_content_length(&response, MAX_RESPONSE_SIZE, ¶ms.url)?;let status = response.status();+ let response_headers = response.headers().clone();let mut response_builder = Response::builder().status(status);- for (header, value) in response.headers() {- response_builder = response_builder.header(header, value);+ for (header, value) in response_headers.iter() {+ if is_allowed_response_header(header) {+ response_builder = response_builder.header(header, value);+ }}let size_limited = apply_size_limit(@@ -91,6 +54,7 @@ pub async fn proxy_request_handler(response_builder.header("Cross-Origin-Resource-Policy", "cross-origin")+ .header("X-Content-Type-Options", "nosniff").body(Body::from_stream(size_limited)).map_err(|e| {tracing::error!(error=?e, "could not stream chunks");@@ -98,6 +62,84 @@ pub async fn proxy_request_handler(})}+async fn fetch_upstream(+ http_client: &reqwest::Client,+ mut url: url::Url,+ headers: &HeaderMap,+) -> Result<reqwest::Response, FetchError> {+ let mut redirects_remaining = MAX_REDIRECTS;++ loop {+ assert_not_internal(&url).await?;+ let response = http_client+ .get(url.as_str())+ .headers(headers.clone())+ .send()+ .await+ .map_err(|e| {+ let error_chain = build_error_chain(&e);+ tracing::warn!(url = %url, error = %error_chain, "upstream proxy request failed");+ if e.is_timeout() {+ FetchError::UpstreamTimeout(error_chain)+ } else if e.is_connect() {+ FetchError::UpstreamConnect(error_chain)+ } else if e.is_redirect() {+ FetchError::UpstreamRedirect(error_chain)+ } else {+ FetchError::UpstreamNetwork(error_chain)+ }+ })?;+ let status = response.status();++ if status.is_redirection() {+ if redirects_remaining == 0 {+ return Err(FetchError::UpstreamRedirect(format!(+ "exceeded maximum of {MAX_REDIRECTS} redirects"+ )));+ }+ let next = redirect_target(&url, &response)?;+ tracing::debug!(from = %url, to = %next, "following proxy redirect");+ redirects_remaining -= 1;+ url = next;+ continue;+ }++ if !status.is_success() {+ return Err(FetchError::UpstreamStatus(status));+ }++ return Ok(response);+ }+}++fn forwarded_request_headers(headers: &HeaderMap) -> HeaderMap {+ headers+ .iter()+ .filter(|(name, _)| is_allowed_request_header(name))+ .map(|(name, value)| (name.clone(), value.clone()))+ .collect()+}++fn is_allowed_request_header(header: &HeaderName) -> bool {+ matches!(header.as_str(), "accept" | "accept-language" | "user-agent")+}++fn is_allowed_response_header(header: &HeaderName) -> bool {+ matches!(+ header.as_str(),+ "cache-control"+ | "content-encoding"+ | "content-language"+ | "content-type"+ | "etag"+ | "expires"+ | "last-modified"+ )+}+pub fn router() -> Router<crate::api::context::ApiContext> {Router::new().route("/", get(proxy_request_handler))}++#[cfg(test)]+mod test;diff --git a/services/unfurl_service/src/api/proxy/test.rs b/services/unfurl_service/src/api/proxy/test.rsnew file mode 100644index 0000000000..e33d946051--- /dev/null+++ b/services/unfurl_service/src/api/proxy/test.rs@@ -0,0 +1,66 @@+use super::*;+use axum::http::header::{AUTHORIZATION, COOKIE, REFERER, SET_COOKIE};++fn redirect_response(location: &str) -> reqwest::Response {+ axum::http::Response::builder()+ .status(302)+ .header("Location", location)+ .body("")+ .unwrap()+ .into()+}++#[test]+fn forwarded_headers_use_a_safe_allowlist() {+ let mut incoming = HeaderMap::new();+ incoming.insert("accept", "image/avif,image/webp,*/*".parse().unwrap());+ incoming.insert("accept-language", "en-US".parse().unwrap());+ incoming.insert("user-agent", "test-browser".parse().unwrap());+ incoming.insert(AUTHORIZATION, "Bearer secret".parse().unwrap());+ incoming.insert(COOKIE, "session=secret".parse().unwrap());+ incoming.insert(REFERER, "https://app.macro.com/private".parse().unwrap());+ incoming.insert("x-forwarded-for", "10.0.0.1".parse().unwrap());++ let forwarded = forwarded_request_headers(&incoming);++ assert_eq!(forwarded.get("accept"), incoming.get("accept"));+ assert_eq!(+ forwarded.get("accept-language"),+ incoming.get("accept-language")+ );+ assert_eq!(forwarded.get("user-agent"), incoming.get("user-agent"));+ assert!(!forwarded.contains_key(AUTHORIZATION));+ assert!(!forwarded.contains_key(COOKIE));+ assert!(!forwarded.contains_key(REFERER));+ assert!(!forwarded.contains_key("x-forwarded-for"));+}++#[test]+fn response_headers_exclude_credentials_and_hop_by_hop_metadata() {+ assert!(is_allowed_response_header(&reqwest::header::CONTENT_TYPE));+ assert!(is_allowed_response_header(&reqwest::header::CACHE_CONTROL));+ assert!(!is_allowed_response_header(&SET_COOKIE));+ assert!(!is_allowed_response_header(&reqwest::header::CONNECTION));+ assert!(!is_allowed_response_header(&reqwest::header::LOCATION));+}++#[test]+fn relative_redirects_remain_supported() {+ let current = url::Url::parse("https://example.com/assets/icon").unwrap();+ let response = redirect_response("../favicon.ico");++ let target = redirect_target(¤t, &response).unwrap();++ assert_eq!(target.as_str(), "https://example.com/favicon.ico");+}++#[tokio::test]+async fn redirect_to_metadata_endpoint_is_rejected_before_following() {+ let current = url::Url::parse("https://example.com/start").unwrap();+ let response = redirect_response("http://169.254.169.254/latest/meta-data/");+ let target = redirect_target(¤t, &response).unwrap();++ let result = assert_not_internal(&target).await;++ assert!(matches!(result, Err(FetchError::PrivateIp)));+}diff --git a/services/unfurl_service/src/api/unfurl/get_unfurl.rs b/services/unfurl_service/src/api/unfurl/get_unfurl.rsindex e595ba5188..5b8ccada63 100644--- a/services/unfurl_service/src/api/unfurl/get_unfurl.rs+++ b/services/unfurl_service/src/api/unfurl/get_unfurl.rs@@ -2,6 +2,7 @@ use axum::extract::Json;use axum::extract::State;use axum::http::StatusCode;+use crate::http_safety::SsrfSafeHttpClient;use crate::unfurl::GetUnfurlResponseList;use crate::unfurl::fetch_links_async;use serde::{Deserialize, Serialize};@@ -31,10 +32,10 @@ pub struct GetUnfurlBulkBody {))]#[tracing::instrument(skip(http_client, body))]pub async fn get_bulk_unfurl_handler(- State(http_client): State<reqwest::Client>,+ State(http_client): State<SsrfSafeHttpClient>,body: Json<GetUnfurlBulkBody>,) -> (StatusCode, Json<GetUnfurlBulkResponse>) {- let links = fetch_links_async(&http_client, &body.url_list).await;+ let links = fetch_links_async(http_client.as_ref(), &body.url_list).await;let response = GetUnfurlBulkResponse { responses: links };(StatusCode::OK, Json(response))}diff --git a/services/unfurl_service/src/http_safety/mod.rs b/services/unfurl_service/src/http_safety/mod.rsindex 2b680a9232..cdbb0c351f 100644--- a/services/unfurl_service/src/http_safety/mod.rs+++ b/services/unfurl_service/src/http_safety/mod.rs@@ -1,11 +1,46 @@+mod resolver;+use axum::response::{IntoResponse, Response};use futures::StreamExt;use reqwest::StatusCode;use std::error::Error;use std::fmt;use std::net::IpAddr;+use std::sync::Arc;use url::Url;+use resolver::PrivateIpFilteringResolver;++/// Maximum redirects followed by unfurl and proxy fetches.+pub const MAX_REDIRECTS: u8 = 5;++const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(8);+const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3);++/// HTTP client with redirect and DNS-rebinding SSRF protections built in.+#[derive(Clone)]+pub struct SsrfSafeHttpClient(reqwest::Client);++impl SsrfSafeHttpClient {+ /// Build a client with automatic redirects disabled and private DNS+ /// connection targets filtered.+ pub fn new() -> Result<Self, reqwest::Error> {+ reqwest::Client::builder()+ .timeout(REQUEST_TIMEOUT)+ .connect_timeout(CONNECT_TIMEOUT)+ .redirect(reqwest::redirect::Policy::none())+ .dns_resolver(Arc::new(PrivateIpFilteringResolver))+ .build()+ .map(Self)+ }+}++impl AsRef<reqwest::Client> for SsrfSafeHttpClient {+ fn as_ref(&self) -> &reqwest::Client {+ &self.0+ }+}+#[derive(Debug)]pub enum FetchError {InvalidUrl(String),@@ -119,7 +154,10 @@ pub fn is_private_ip(ip: &IpAddr) -> bool {if let Some(mapped_v4) = v6.to_ipv4_mapped() {return is_private_ip(&IpAddr::V4(mapped_v4));}- v6.is_loopback() || v6.is_unspecified()+ v6.is_loopback()+ || v6.is_unspecified()+ || v6.is_unique_local()+ || v6.is_unicast_link_local()}}}@@ -128,7 +166,7 @@ pub async fn send_request(http_client: &reqwest::Client,url: &Url,) -> Result<reqwest::Response, FetchError> {- let response = http_client.get(url.as_str()).send().await.map_err(|e| {+ http_client.get(url.as_str()).send().await.map_err(|e| {let error_chain = build_error_chain(&e);tracing::warn!(url = %url, error = %error_chain, "upstream request failed");if e.is_timeout() {@@ -140,13 +178,30 @@ pub async fn send_request(} else {FetchError::UpstreamNetwork(error_chain)}+ })+}++/// Resolve a redirect's `Location` header against the current URL.+pub fn redirect_target(current: &Url, response: &reqwest::Response) -> Result<Url, FetchError> {+ let location = response+ .headers()+ .get(reqwest::header::LOCATION)+ .ok_or_else(|| {+ FetchError::UpstreamRedirect("redirect response missing Location header".into())+ })?+ .to_str()+ .map_err(|e| FetchError::UpstreamRedirect(format!("invalid Location header: {e}")))?;++ let mut next = current.join(location).map_err(|e| {+ FetchError::UpstreamRedirect(format!("could not parse redirect target {location}: {e}"))})?;- if !response.status().is_success() {- return Err(FetchError::UpstreamStatus(response.status()));+ if next.scheme() != "http" && next.scheme() != "https" {+ return Err(FetchError::InvalidScheme);}+ next.set_fragment(None);- Ok(response)+ Ok(next)}pub fn content_type_of(response: &reqwest::Response) -> String {diff --git a/services/unfurl_service/src/http_safety/resolver.rs b/services/unfurl_service/src/http_safety/resolver.rsnew file mode 100644index 0000000000..489dd05933--- /dev/null+++ b/services/unfurl_service/src/http_safety/resolver.rs@@ -0,0 +1,47 @@+//! DNS resolver that rejects private and internal connection targets.++use std::net::SocketAddr;++use reqwest::dns::{Addrs, Name, Resolve, Resolving};++use super::is_private_ip;++pub(super) struct PrivateIpFilteringResolver;++impl Resolve for PrivateIpFilteringResolver {+ fn resolve(&self, name: Name) -> Resolving {+ Box::pin(async move {+ let hostname = name.as_str().to_owned();+ let lookup = tokio::net::lookup_host(format!("{hostname}:0"))+ .await+ .map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)?;++ let filtered: Vec<SocketAddr> = lookup+ .filter(|address| {+ let allowed = !is_private_ip(&address.ip());+ if !allowed {+ tracing::warn!(+ host = %hostname,+ ip = %address.ip(),+ "dropped resolved private/internal IP from connection candidates"+ );+ }+ allowed+ })+ .collect();++ if filtered.is_empty() {+ return Err(Box::new(std::io::Error::new(+ std::io::ErrorKind::PermissionDenied,+ format!(+ "all resolved IPs for {hostname} are private/internal; refusing to connect"+ ),+ ))+ as Box<dyn std::error::Error + Send + Sync>);+ }++ let addresses: Addrs = Box::new(filtered.into_iter());+ Ok(addresses)+ })+ }+}diff --git a/services/unfurl_service/src/http_safety/test.rs b/services/unfurl_service/src/http_safety/test.rsindex c4913323b0..caff936213 100644--- a/services/unfurl_service/src/http_safety/test.rs+++ b/services/unfurl_service/src/http_safety/test.rs@@ -62,6 +62,13 @@ fn test_is_private_ip_unspecified() {assert!(is_private_ip(&"::".parse().unwrap()));}+#[test]+fn test_is_private_ip_v6_unique_and_link_local() {+ assert!(is_private_ip(&"fc00::1".parse().unwrap()));+ assert!(is_private_ip(&"fd12:3456:789a::1".parse().unwrap()));+ assert!(is_private_ip(&"fe80::1".parse().unwrap()));+}+#[test]fn test_is_private_ip_mapped_v4() {assert!(is_private_ip(&"::ffff:127.0.0.1".parse().unwrap()));@@ -78,6 +85,40 @@ fn test_is_private_ip_public() {assert!(!is_private_ip(&"2607:f8b0:4004:800::200e".parse().unwrap()));}+fn redirect_response(location: Option<&str>) -> reqwest::Response {+ let mut builder = axum::http::Response::builder().status(302);+ if let Some(location) = location {+ builder = builder.header("Location", location);+ }+ builder.body("").unwrap().into()+}++#[test]+fn test_redirect_target_supports_absolute_and_relative_locations() {+ let current = Url::parse("https://example.com/a/page").unwrap();++ let relative = redirect_target(¤t, &redirect_response(Some("../next"))).unwrap();+ assert_eq!(relative.as_str(), "https://example.com/next");++ let absolute = redirect_target(+ ¤t,+ &redirect_response(Some("https://cdn.example.com/final#fragment")),+ )+ .unwrap();+ assert_eq!(absolute.as_str(), "https://cdn.example.com/final");+}++#[test]+fn test_redirect_target_rejects_unsafe_or_missing_locations() {+ let current = Url::parse("https://example.com/start").unwrap();++ let unsafe_scheme = redirect_target(¤t, &redirect_response(Some("file:///etc/passwd")));+ assert!(matches!(unsafe_scheme, Err(FetchError::InvalidScheme)));++ let missing = redirect_target(¤t, &redirect_response(None));+ assert!(matches!(missing, Err(FetchError::UpstreamRedirect(_))));+}+#[tokio::test]async fn test_assert_not_internal_blocks_loopback() {let url = Url::parse("http://127.0.0.1/secret").unwrap();diff --git a/services/unfurl_service/src/main.rs b/services/unfurl_service/src/main.rsindex ee620f3cd0..a8af5e1d13 100644--- a/services/unfurl_service/src/main.rs+++ b/services/unfurl_service/src/main.rs@@ -20,12 +20,11 @@ async fn main() -> anyhow::Result<()> {tracing::trace!("initialized config");- let http_client = reqwest::Client::builder()- .timeout(std::time::Duration::from_secs(8))- .connect_timeout(std::time::Duration::from_secs(3))- .redirect(reqwest::redirect::Policy::limited(5))- .build()- .context("failed to build http client")?;+ // The shared client serves the legacy bulk-unfurl and proxy routes.+ // Redirects are handled manually so every hop can be checked, while the+ // custom resolver rejects DNS rebinding at connection time.+ let http_client = http_safety::SsrfSafeHttpClient::new()+ .context("failed to build shared safe http client")?;let state = api::context::ApiContext {environment: config.environment,diff --git a/services/unfurl_service/src/unfurl/mod.rs b/services/unfurl_service/src/unfurl/mod.rsindex b954a8323b..c988a19c10 100644--- a/services/unfurl_service/src/unfurl/mod.rs+++ b/services/unfurl_service/src/unfurl/mod.rs@@ -5,8 +5,8 @@ use std::collections::HashMap;use url::Url;use crate::http_safety::{- FetchError, assert_not_internal, check_content_length, content_type_of, send_request,- validate_url,+ FetchError, MAX_REDIRECTS, assert_not_internal, check_content_length, content_type_of,+ redirect_target, send_request, validate_url,};pub use ::unfurl::domain::favicon::append_optimistic_favico;@@ -118,10 +118,34 @@ pub async fn extract_meta_tags_prod(client: &reqwest::Client,raw_url: &str,) -> Result<HashMap<String, String>, UnfurlFetchError> {- let url = validate_url(raw_url)?;- assert_not_internal(&url).await?;+ let mut url = validate_url(raw_url)?;+ let mut redirects_remaining = MAX_REDIRECTS;++ let response = loop {+ assert_not_internal(&url).await?;+ let response = send_request(client, &url).await?;+ let status = response.status();++ if status.is_redirection() {+ if redirects_remaining == 0 {+ return Err(FetchError::UpstreamRedirect(format!(+ "exceeded maximum of {MAX_REDIRECTS} redirects"+ ))+ .into());+ }+ let next = redirect_target(&url, &response)?;+ tracing::debug!(from = %url, to = %next, "following redirect");+ redirects_remaining -= 1;+ url = next;+ continue;+ }++ if !status.is_success() {+ return Err(FetchError::UpstreamStatus(status).into());+ }- let response = send_request(client, &url).await?;+ break response;+ };let content_type = content_type_of(&response);if !is_html_content_type(&content_type) {