A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
use super::{
ExternalBody, ExternalFetchError, ExternalSuccessStatus, FetchedExternal,
byte_range::{ExternalRangeSelection, validate_external_partial},
is_redirect_status,
response::{ExternalStreamDecision, external_stream_decision},
};
use crate::{
byte_budget::{BudgetedBytes, ByteBudget, ByteReservation},
constants::{self, AssetExtension},
disposition::PUBLIC_MEDIA_MAX_HEADER_VALUE_BYTES,
http_headers, metrics, mime, public_net_policy,
request_log::{self, Stage},
response_body_limit,
server::{format_policy::image_extension_from_filename, params::url_filename, state::AppState},
};
use axum::http::{HeaderMap, HeaderName, Method, StatusCode, header};
use bytes::Bytes;
use tracing::warn;
const EXTERNAL_FETCH_REDIRECT_LIMIT: usize = 5;
const DEFAULT_EXTERNAL_CONTENT_TYPE: &str = "application/octet-stream";
const EXTERNAL_SNIFF_PREFIX_BYTES: usize = 8192;
#[derive(Clone, Copy, Eq, PartialEq)]
pub(super) enum ExternalFetchMode {
Buffered,
Streaming,
}
impl ExternalFetchMode {
fn allows_streaming(self) -> bool {
self == Self::Streaming
}
}
pub(super) struct ExternalFetchRequest<'a> {
pub(super) app: &'a AppState,
pub(super) url: &'a str,
pub(super) range: Option<ExternalRangeSelection<'a>>,
pub(super) mode: ExternalFetchMode,
}
pub(super) async fn fetch_external_with_range(
request: ExternalFetchRequest<'_>,
) -> Result<FetchedExternal, ExternalFetchError> {
let start_ms = metrics::now_ms();
let result = fetch_external_inner(request).await;
request_log::record_stage(Stage::Fetch, (metrics::now_ms() - start_ms).max(0) as u64);
result
}
struct ExternalUpstream {
url: String,
response: reqwest::Response,
}
async fn send_external_request(
app: &AppState,
url: &str,
method: Method,
range: Option<ExternalRangeSelection<'_>>,
) -> Result<ExternalUpstream, ExternalFetchError> {
let external_metrics = app.metrics.external();
let mut current_url = url.to_owned();
let mut visited: Vec<String> = Vec::new();
visited
.try_reserve_exact(EXTERNAL_FETCH_REDIRECT_LIMIT + 1)
.map_err(|_| ExternalFetchError::BufferAllocationFailed)?;
for _ in 0..=EXTERNAL_FETCH_REDIRECT_LIMIT {
if visited.iter().any(|seen| seen == ¤t_url) {
warn!(url = %public_net_policy::external_url_for_log(¤t_url), "redirect loop detected");
return Err(ExternalFetchError::TooManyRedirects);
}
visited.push(current_url.clone());
if let Err(err) = public_net_policy::validate_url(¤t_url) {
warn!(?err, url = %public_net_policy::external_url_for_log(¤t_url), "blocked external fetch");
external_metrics.blocked_url_attempts.inc();
return Err(ExternalFetchError::BlockedUrl);
}
let mut outbound = app
.media
.external_client()
.request(method.clone(), ¤t_url);
if let Some(range) = range {
outbound = outbound.header(header::RANGE, range.header_value());
}
let response = outbound.send().await.map_err(|err| {
let err = err.without_url();
if public_net_policy::is_pinned_dns_failure(&err) {
warn!(?err, url = %public_net_policy::external_url_for_log(¤t_url), "blocked external fetch");
external_metrics.blocked_url_attempts.inc();
return ExternalFetchError::BlockedUrl;
}
warn!(url = %public_net_policy::external_url_for_log(¤t_url), %err, "external send failed");
external_metrics.fetch_failures.inc();
ExternalFetchError::FetchFailed
})?;
let status = response.status();
if is_redirect_status(status) {
let location = match bounded_single_visible_header(
response.headers(),
&header::LOCATION,
) {
Some(location) if !location.is_empty() => location,
Some(_) | None => {
warn!(url = %public_net_policy::external_url_for_log(¤t_url), status = status.as_u16(), "redirect has an invalid Location");
external_metrics.fetch_failures.inc();
return Err(ExternalFetchError::FetchFailed);
}
};
current_url =
public_net_policy::resolve_redirect(¤t_url, location).map_err(|err| {
warn!(url = %public_net_policy::external_url_for_log(¤t_url), location = %public_net_policy::external_url_for_log(location), ?err, "redirect target blocked");
external_metrics.blocked_url_attempts.inc();
ExternalFetchError::BlockedUrl
})?;
continue;
}
if !status.is_success() {
return Err(ExternalFetchError::UpstreamFailure(status));
}
return Ok(ExternalUpstream {
url: current_url,
response,
});
}
Err(ExternalFetchError::TooManyRedirects)
}
pub(super) struct ExternalHead {
pub(super) url: String,
pub(super) status: StatusCode,
pub(super) content_type: String,
pub(super) content_length: Option<u64>,
}
pub(super) async fn fetch_external_head(
app: &AppState,
url: &str,
) -> Result<ExternalHead, ExternalFetchError> {
let start_ms = metrics::now_ms();
let result = send_external_request(app, url, Method::HEAD, None).await;
request_log::record_stage(Stage::Fetch, (metrics::now_ms() - start_ms).max(0) as u64);
let ExternalUpstream { url, response } = result?;
Ok(ExternalHead {
url,
status: response.status(),
content_type: external_content_type(response.headers()),
content_length: validated_content_length(response.headers()),
})
}
async fn fetch_external_inner(
request: ExternalFetchRequest<'_>,
) -> Result<FetchedExternal, ExternalFetchError> {
let ExternalFetchRequest {
app,
url,
range,
mode,
} = request;
let external_metrics = app.metrics.external();
let ExternalUpstream {
url: current_url,
mut response,
} = send_external_request(app, url, Method::GET, range).await?;
let content_type = external_content_type(response.headers());
let declared_length = validated_content_length(response.headers());
if let Some(len) = declared_length
&& len > constants::MAX_MEDIA_PROXY_BYTES as u64
{
warn!(url = %public_net_policy::external_url_for_log(¤t_url), len, "external payload too large");
return Err(ExternalFetchError::PayloadTooLarge);
}
let status = if response.status() == StatusCode::PARTIAL_CONTENT {
let content_range =
bounded_single_visible_header(response.headers(), &header::CONTENT_RANGE);
let Some(partial) = validate_external_partial(
range,
content_range,
declared_length,
constants::MAX_MEDIA_PROXY_BYTES,
) else {
warn!(url = %public_net_policy::external_url_for_log(¤t_url), "invalid upstream partial response");
external_metrics.fetch_failures.inc();
return Err(ExternalFetchError::FetchFailed);
};
ExternalSuccessStatus::Partial(partial)
} else {
ExternalSuccessStatus::Complete
};
let mut prefix = Bytes::new();
if let ExternalStreamDecision::Stream(content_length) = external_stream_decision(
mode.allows_streaming(),
response.content_length(),
&content_type,
) {
prefix = external_body_prefix(&mut response, ¤t_url, &external_metrics).await?;
if !external_streamed_source_is_svg(&prefix, &url_filename(¤t_url)) {
return Ok(FetchedExternal {
url: current_url,
status,
body: ExternalBody::Streaming {
response,
prefix,
content_length,
},
content_type,
});
}
}
let data = buffer_external_response(ExternalBufferRequest {
response,
prefix,
url: ¤t_url,
budget: app.media.external_buffer_bytes(),
metrics: &external_metrics,
content_length: declared_length,
limit: constants::MAX_MEDIA_PROXY_BYTES,
})
.await?;
if status
.partial()
.and_then(|partial| partial.content_length())
.is_some_and(|expected| data.len() as u64 != expected)
{
warn!(url = %public_net_policy::external_url_for_log(¤t_url), "upstream partial response body length mismatch");
external_metrics.fetch_failures.inc();
return Err(ExternalFetchError::FetchFailed);
}
Ok(FetchedExternal {
url: current_url,
status,
body: ExternalBody::Buffered(data),
content_type,
})
}
fn external_content_type(headers: &HeaderMap) -> String {
bounded_single_visible_header(headers, &header::CONTENT_TYPE)
.filter(|content_type| !content_type.is_empty())
.unwrap_or(DEFAULT_EXTERNAL_CONTENT_TYPE)
.to_owned()
}
async fn external_body_prefix(
response: &mut reqwest::Response,
url: &str,
metrics: &metrics::external::ExternalMetrics,
) -> Result<Bytes, ExternalFetchError> {
let mut prefix: Vec<u8> = Vec::new();
prefix
.try_reserve_exact(EXTERNAL_SNIFF_PREFIX_BYTES)
.map_err(|_| ExternalFetchError::BufferAllocationFailed)?;
let mut empty_chunks_remaining =
response_body_limit::response_body_chunk_limit(constants::MAX_MEDIA_PROXY_BYTES as u64);
while prefix.len() < EXTERNAL_SNIFF_PREFIX_BYTES {
let Some(chunk) = response.chunk().await.map_err(|err| {
let err = err.without_url();
warn!(url = %public_net_policy::external_url_for_log(url), %err, "external body read failed");
metrics.fetch_failures.inc();
ExternalFetchError::FetchFailed
})?
else {
break;
};
if chunk.is_empty() {
empty_chunks_remaining = empty_chunks_remaining.checked_sub(1).ok_or_else(|| {
warn!(url = %public_net_policy::external_url_for_log(url), "external sniff prefix exceeded its empty chunk limit");
ExternalFetchError::PayloadTooLarge
})?;
continue;
}
prefix.extend_from_slice(&chunk);
}
Ok(Bytes::from(prefix))
}
pub(super) fn external_streamed_source_is_svg(prefix: &[u8], filename: &str) -> bool {
image_extension_from_filename(filename) == Some(AssetExtension::Svg)
|| mime::sniff(&prefix[..prefix.len().min(EXTERNAL_SNIFF_PREFIX_BYTES)]).mime
== "image/svg+xml"
}
pub(super) async fn buffer_external_body(
app: &AppState,
url: &str,
response: reqwest::Response,
prefix: Bytes,
content_length: Option<u64>,
) -> Result<BudgetedBytes, ExternalFetchError> {
let external_metrics = app.metrics.external();
buffer_external_response(ExternalBufferRequest {
response,
prefix,
url,
budget: app.media.external_buffer_bytes(),
metrics: &external_metrics,
content_length,
limit: constants::MAX_MEDIA_PROXY_BYTES,
})
.await
}
fn bounded_single_visible_header<'a>(headers: &'a HeaderMap, name: &HeaderName) -> Option<&'a str> {
let mut values = headers.get_all(name).iter();
let value = values.next()?;
if values.next().is_some() || value.as_bytes().len() > PUBLIC_MEDIA_MAX_HEADER_VALUE_BYTES {
return None;
}
value.to_str().ok()
}
fn validated_content_length(headers: &HeaderMap) -> Option<u64> {
let mut values = headers.get_all(header::CONTENT_LENGTH).iter();
let value = values.next()?;
if values.next().is_some() || value.as_bytes().len() > PUBLIC_MEDIA_MAX_HEADER_VALUE_BYTES {
return None;
}
http_headers::parse_content_length(headers)
}
pub(super) struct ExternalBufferRequest<'a> {
pub(super) response: reqwest::Response,
pub(super) prefix: Bytes,
pub(super) url: &'a str,
pub(super) budget: &'a ByteBudget,
pub(super) metrics: &'a metrics::external::ExternalMetrics,
pub(super) content_length: Option<u64>,
pub(super) limit: usize,
}
pub(super) async fn buffer_external_response(
request: ExternalBufferRequest<'_>,
) -> Result<BudgetedBytes, ExternalFetchError> {
let ExternalBufferRequest {
mut response,
prefix,
url,
budget,
metrics,
content_length,
limit,
} = request;
let declared_length = match content_length {
Some(length) if length > limit as u64 => {
warn!(url = %public_net_policy::external_url_for_log(url), length, "external payload too large");
return Err(ExternalFetchError::PayloadTooLarge);
}
Some(length) => {
Some(usize::try_from(length).expect("a bounded external content length fits usize"))
}
None => None,
};
let initial_capacity = declared_length.unwrap_or(0).max(prefix.len());
let _transport_chunk_reservation = reserve(
budget,
metrics,
response_body_limit::RESPONSE_BODY_TRANSPORT_CHUNK_BYTES_MAX,
)?;
let mut reservation = reserve(budget, metrics, initial_capacity)?;
let mut buf: Vec<u8> = Vec::new();
buf.try_reserve_exact(initial_capacity)
.map_err(|_| ExternalFetchError::BufferAllocationFailed)?;
let mut reserved_bytes =
grow_to_capacity(&mut reservation, metrics, buf.capacity(), initial_capacity)?;
buf.extend_from_slice(&prefix);
let mut empty_chunks_remaining = response_body_limit::response_body_chunk_limit(limit as u64);
while let Some(chunk) = response.chunk().await.map_err(|err| {
let err = err.without_url();
warn!(url = %public_net_policy::external_url_for_log(url), %err, "external body read failed");
metrics.fetch_failures.inc();
ExternalFetchError::FetchFailed
})? {
if chunk.is_empty() {
empty_chunks_remaining = empty_chunks_remaining.checked_sub(1).ok_or_else(|| {
warn!(url = %public_net_policy::external_url_for_log(url), "external payload exceeded its empty chunk limit");
ExternalFetchError::PayloadTooLarge
})?;
continue;
}
let Some(next_len) = buf
.len()
.checked_add(chunk.len())
.filter(|len| *len <= limit)
else {
warn!(url = %public_net_policy::external_url_for_log(url), "external payload too large");
return Err(ExternalFetchError::PayloadTooLarge);
};
if next_len > buf.capacity() {
let target_capacity = buf
.capacity()
.checked_mul(2)
.expect("a bounded external buffer capacity doubling fits usize")
.max(next_len)
.min(limit);
let additional = target_capacity
.checked_sub(reserved_bytes)
.expect("an external buffer target capacity covers its reservation");
if !reservation.try_grow(additional) {
metrics.external_buffer_rejected.inc();
return Err(ExternalFetchError::BufferBudgetExhausted);
}
reserved_bytes = target_capacity;
buf.try_reserve_exact(target_capacity - buf.len())
.map_err(|_| ExternalFetchError::BufferAllocationFailed)?;
reserved_bytes =
grow_to_capacity(&mut reservation, metrics, buf.capacity(), reserved_bytes)?;
}
buf.extend_from_slice(&chunk);
}
if let Some(expected_length) = declared_length
&& buf.len() != expected_length
{
warn!(
url = %public_net_policy::external_url_for_log(url),
expected_length,
actual_length = buf.len(),
"external buffered body length did not match Content-Length"
);
metrics.fetch_failures.inc();
return Err(ExternalFetchError::FetchFailed);
}
reservation.shrink_to(buf.capacity());
Ok(BudgetedBytes::budgeted(Bytes::from(buf), reservation))
}
fn reserve(
budget: &ByteBudget,
metrics: &metrics::external::ExternalMetrics,
amount: usize,
) -> Result<ByteReservation, ExternalFetchError> {
budget.try_reserve(amount).ok_or_else(|| {
metrics.external_buffer_rejected.inc();
ExternalFetchError::BufferBudgetExhausted
})
}
fn grow_to_capacity(
reservation: &mut ByteReservation,
metrics: &metrics::external::ExternalMetrics,
capacity: usize,
reserved_bytes: usize,
) -> Result<usize, ExternalFetchError> {
if capacity <= reserved_bytes {
return Ok(reserved_bytes);
}
if !reservation.try_grow(capacity - reserved_bytes) {
metrics.external_buffer_rejected.inc();
return Err(ExternalFetchError::BufferBudgetExhausted);
}
Ok(capacity)
}