A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
pub(in crate::server) mod cache;
pub mod cache_key;
pub(in crate::server) mod execution;
pub mod parameters;
use crate::{
byte_budget::{BudgetedBytes, ByteBudget},
byte_cache::ByteCacheSettings,
coalescer::ByteCoalescer,
config::Config,
constants::AssetExtension,
image_quality::ImageQuality,
image_transform::{AnimationLimits, AnimationMode, ImageOptions, ResizeMode},
media_limits::MediaLimits,
media_process::{MediaBytes, MediaError},
metrics::{self, transform::TransformMetrics},
mime,
output_format::OutputFormat,
server::{
format_policy::{
OriginalImageRequest, content_type_is_trustworthy, default_transform_quality,
effective_animated_image_output_format, extension_from_mime,
external_default_output_extension, image_extension_from_filename,
is_browser_displayable_image, is_svg_content_type, is_untransformed_fallback_image,
same_format_loaded_image_request_can_use_original_with_sniff, source_image_format,
transform_response_content_type, transform_static_quality_default,
},
native_task_executor::{NativeTaskExecutor, NativeTaskExecutorSettings},
response::{
GeneratedMediaResponse, MediaResponse, content_disposition_header,
error::{text_with_reason, text_with_source},
generated_media_response, media_response,
},
transform::{
cache::{
CachedTransformRequest, cached_transform, cached_transform_hit,
coalescer_error_is_transient, coalescer_failure_response, forbid_storing,
},
cache_key::{TransformCacheKeyInput, transform_cache_key},
execution::{
VideoTransformOptions, coalesced_work_result, deadline_instant, run_transform,
run_video_transform,
},
parameters::{
TransformRoute, ValidatedTransformParameters, transform_parameter_error_response,
validate_transform_parameters,
},
},
},
transform_cache::{TransformCache, TransformCacheSettings},
};
use axum::{
http::{HeaderMap, Method, StatusCode, header},
response::Response,
};
use bytes::Bytes;
use std::{collections::HashMap, sync::Arc, time::Duration};
const NATIVE_MEMORY_HEADROOM_BYTES: usize = 512 << 20;
const NATIVE_MEMORY_FLOOR_BYTES: usize = 128 << 20;
const CGROUP_MEMORY_LIMIT_PATHS: [&str; 2] = [
"/sys/fs/cgroup/memory.max",
"/sys/fs/cgroup/memory/memory.limit_in_bytes",
];
const CONTENT_TYPE_SNIFF_PREFIX_BYTES: usize = 8192;
const FAILED_TRANSFORM_ENTRIES: u64 = 4096;
pub(in crate::server) struct TransformRuntime {
limits: MediaLimits,
animation: AnimationLimits,
transform_timeout_ms: u64,
tasks: NativeTaskExecutor,
cache: TransformCache,
failed_transforms: moka::sync::Cache<String, ()>,
metrics: Arc<TransformMetrics>,
}
pub(in crate::server) struct ServeBytesRequest<'a> {
pub(in crate::server) method: Method,
pub(in crate::server) data: Bytes,
pub(in crate::server) content_type: String,
pub(in crate::server) cache_identity: &'a str,
pub(in crate::server) filename: &'a str,
pub(in crate::server) route: TransformRoute,
pub(in crate::server) params: &'a HashMap<String, String>,
pub(in crate::server) headers: &'a HeaderMap,
}
impl TransformRuntime {
pub(in crate::server) fn new(
cfg: &Config,
metrics: &Arc<metrics::Metrics>,
) -> anyhow::Result<Self> {
let memory_limit_bytes = process_memory_limit_bytes();
let memory_budget_bytes = native_memory_budget_bytes(
memory_limit_bytes,
cfg.media.transform_cache_capacity_bytes,
);
let limits =
MediaLimits::default_from_config().with_native_memory_budget(memory_budget_bytes);
tracing::info!(
memory_limit_bytes,
transform_cache_bytes = cfg.media.transform_cache_capacity_bytes,
memory_budget_bytes,
max_native_transforms = cfg.media.max_native_transforms,
"native memory budget"
);
Ok(Self {
limits,
animation: AnimationLimits::new(
cfg.media.max_encode_frames,
cfg.media.max_encode_duration_ms,
)?,
transform_timeout_ms: cfg.media.transform_timeout_ms,
tasks: NativeTaskExecutor::new(NativeTaskExecutorSettings {
max_native_transforms: cfg.media.max_native_transforms,
worker_queue_capacity: cfg.media.worker_queue_capacity,
memory_budget_bytes,
native_metrics: metrics.native_transform(),
transform_metrics: metrics.transform(),
}),
cache: TransformCache::new(TransformCacheSettings {
cache: ByteCacheSettings::clamped(
cfg.media.transform_cache_capacity_bytes,
cfg.media.transform_cache_max_entry_bytes,
cfg.media.transform_cache_ttl_ms,
),
budget: ByteBudget::new(usize::MAX),
max_in_flight: ByteCoalescer::UNBOUNDED_CAPACITY,
max_waiters: ByteCoalescer::UNBOUNDED_CAPACITY,
cache_metrics: metrics.transform_cache(),
coalescer_metrics: metrics.coalescer(),
}),
failed_transforms: moka::sync::Cache::builder()
.max_capacity(FAILED_TRANSFORM_ENTRIES)
.time_to_live(Duration::from_millis(
cfg.media.transform_cache_ttl_ms.max(1),
))
.build(),
metrics: metrics.transform(),
})
}
pub(in crate::server) fn limits(&self) -> MediaLimits {
self.limits
}
pub(in crate::server) fn animation(&self) -> AnimationLimits {
self.animation
}
pub(in crate::server) fn cache(&self) -> &TransformCache {
&self.cache
}
pub(in crate::server) fn failed_transforms(&self) -> &moka::sync::Cache<String, ()> {
&self.failed_transforms
}
pub(in crate::server) fn tasks(&self) -> &NativeTaskExecutor {
&self.tasks
}
pub(in crate::server) fn metrics(&self) -> Arc<TransformMetrics> {
Arc::clone(&self.metrics)
}
pub(in crate::server) fn transform_deadline_ms(&self) -> Option<i64> {
Some(metrics::now_ms() + self.transform_timeout_ms as i64)
}
}
fn native_memory_budget_bytes(memory_limit_bytes: usize, transform_cache_bytes: usize) -> usize {
let cache = transform_cache_bytes.min(memory_limit_bytes / 2);
let headroom = NATIVE_MEMORY_HEADROOM_BYTES.min(memory_limit_bytes / 8);
memory_limit_bytes
.saturating_sub(cache)
.saturating_sub(headroom)
.max(NATIVE_MEMORY_FLOOR_BYTES)
}
fn process_memory_limit_bytes() -> usize {
let physical = physical_memory_bytes();
CGROUP_MEMORY_LIMIT_PATHS
.iter()
.find_map(|path| {
std::fs::read_to_string(path)
.ok()?
.trim()
.parse::<usize>()
.ok()
})
.filter(|limit| *limit < physical)
.unwrap_or(physical)
}
fn physical_memory_bytes() -> usize {
let pages = unsafe { libc::sysconf(libc::_SC_PHYS_PAGES) };
let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
usize::try_from(pages)
.unwrap_or(0)
.saturating_mul(usize::try_from(page_size).unwrap_or(0))
}
pub(in crate::server) async fn serve_bytes_or_transform(
runtime: &TransformRuntime,
request: ServeBytesRequest<'_>,
) -> Response {
let ServeBytesRequest {
method,
data,
content_type,
cache_identity,
filename,
route,
params,
headers,
} = request;
let transform = match validate_transform_parameters(params, &runtime.limits(), route) {
Ok(transform) => transform,
Err(error) => return transform_parameter_error_response(error),
};
let prefix = &data[..data.len().min(CONTENT_TYPE_SNIFF_PREFIX_BYTES)];
let content_type = if mime::sniff(prefix).mime == "image/svg+xml" {
"image/svg+xml".to_owned()
} else if content_type_is_trustworthy(&content_type) {
content_type
} else {
mime::detect(prefix, filename, Some(&content_type))
};
let source_is_svg = is_svg_content_type(&content_type)
|| image_extension_from_filename(filename) == Some(AssetExtension::Svg);
let range_header = headers.get(header::RANGE).and_then(|v| v.to_str().ok());
if !source_is_svg && !transform.has_transform_request {
return original_bytes_response(OriginalBytes {
method,
data,
content_type: &content_type,
filename,
range_header,
requested_download: transform.requested_download,
});
}
match mime::category(&content_type) {
Some(mime::Category::Video) => {
serve_video_transform(
runtime,
VideoTransformRequest {
method,
data,
content_type: &content_type,
cache_identity,
filename,
route,
transform,
range_header,
},
)
.await
}
Some(mime::Category::Image) => {
serve_image_transform(
runtime,
ImageTransformRequest {
method,
data,
content_type: &content_type,
cache_identity,
filename,
route,
transform,
range_header,
},
)
.await
}
_ => {
if route == TransformRoute::Attachment && transform.explicit_format.is_some() {
return text_with_reason(
StatusCode::BAD_REQUEST,
"Bad Request",
"media_transform_unsupported",
);
}
original_bytes_response(OriginalBytes {
method,
data,
content_type: &content_type,
filename,
range_header,
requested_download: transform.requested_download,
})
}
}
}
struct OriginalBytes<'a> {
method: Method,
data: Bytes,
content_type: &'a str,
filename: &'a str,
range_header: Option<&'a str>,
requested_download: bool,
}
fn original_bytes_response(request: OriginalBytes<'_>) -> Response {
let disposition = Some(content_disposition_header(
request.content_type,
request.requested_download,
Some(request.filename),
));
media_response(MediaResponse {
method: request.method,
data: request.data.into(),
content_type: request.content_type,
range_header: request.range_header,
disposition,
})
}
struct VideoTransformPlan {
format: OutputFormat,
quality: ImageQuality,
cache_key: String,
}
fn video_transform_plan(
route: TransformRoute,
cache_identity: &str,
transform: &ValidatedTransformParameters,
requested_format: AssetExtension,
) -> VideoTransformPlan {
let format = OutputFormat::coerce_from_extension(requested_format);
let quality = transform.quality.unwrap_or(ImageQuality::High);
VideoTransformPlan {
format,
quality,
cache_key: transform_cache_key(TransformCacheKeyInput {
route,
asset_kind: None,
cache_identity,
width: transform.width,
height: transform.height,
format,
quality: Some(quality),
animated: transform.animated,
effort: None,
resize_mode: None,
}),
}
}
pub(in crate::server) struct CachedVideoTransformRequest<'a> {
pub(in crate::server) method: Method,
pub(in crate::server) cache_identity: &'a str,
pub(in crate::server) filename: &'a str,
pub(in crate::server) route: TransformRoute,
pub(in crate::server) params: &'a HashMap<String, String>,
}
pub(in crate::server) fn cached_video_transform_response(
runtime: &TransformRuntime,
request: CachedVideoTransformRequest<'_>,
) -> Option<Response> {
let transform =
validate_transform_parameters(request.params, &runtime.limits(), request.route).ok()?;
let plan = video_transform_plan(
request.route,
request.cache_identity,
&transform,
transform.explicit_format?,
);
let hit = cached_transform_hit(runtime, &plan.cache_key)?;
Some(transformed_response(TransformedBytes {
method: request.method,
data: hit.data,
content_type: hit.format.mime(),
filename: request.filename,
requested_download: transform.requested_download,
}))
}
struct VideoTransformRequest<'a> {
method: Method,
data: Bytes,
content_type: &'a str,
cache_identity: &'a str,
filename: &'a str,
route: TransformRoute,
transform: ValidatedTransformParameters,
range_header: Option<&'a str>,
}
async fn serve_video_transform(
runtime: &TransformRuntime,
request: VideoTransformRequest<'_>,
) -> Response {
let VideoTransformRequest {
method,
data,
content_type,
cache_identity,
filename,
route,
transform,
range_header,
} = request;
let Some(requested_format) = transform.explicit_format else {
if route == TransformRoute::Attachment {
return text_with_reason(
StatusCode::BAD_REQUEST,
"Bad Request",
"video_transform_format_required",
);
}
return original_bytes_response(OriginalBytes {
method,
data,
content_type,
filename,
range_header,
requested_download: transform.requested_download,
});
};
let VideoTransformPlan {
format,
quality,
cache_key,
} = video_transform_plan(route, cache_identity, &transform, requested_format);
if let Some(hit) = cached_transform_hit(runtime, &cache_key) {
return transformed_response(TransformedBytes {
method,
data: hit.data,
content_type: hit.format.mime(),
filename,
requested_download: transform.requested_download,
});
}
let deadline_ms = runtime.transform_deadline_ms();
let transformed = match cached_transform(CachedTransformRequest {
runtime,
cache_key,
format,
deadline: deadline_instant(deadline_ms),
work: move || async move {
coalesced_work_result(
run_video_transform(
runtime,
data,
VideoTransformOptions {
format,
width: transform.width,
height: transform.height,
quality,
deadline_ms,
},
)
.await,
)
.map(|media| MediaBytes::from(media.bytes))
},
})
.await
{
Ok(bytes) => bytes,
Err(error) => {
if let Some(response) = coalescer_failure_response(error, "coalescer_timeout_video") {
return response;
}
return text_with_source(
StatusCode::BAD_REQUEST,
"Bad Request",
"video_transform_failed",
format!(
"fmt={} w={:?} h={:?} q={}",
format.extension(),
transform.width,
transform.height,
quality
),
);
}
};
transformed_response(TransformedBytes {
method,
data: transformed,
content_type: format.mime(),
filename,
requested_download: transform.requested_download,
})
}
struct ImageTransformRequest<'a> {
method: Method,
data: Bytes,
content_type: &'a str,
cache_identity: &'a str,
filename: &'a str,
route: TransformRoute,
transform: ValidatedTransformParameters,
range_header: Option<&'a str>,
}
async fn serve_image_transform(
runtime: &TransformRuntime,
request: ImageTransformRequest<'_>,
) -> Response {
let ImageTransformRequest {
method,
data,
content_type,
cache_identity,
filename,
route,
transform,
range_header,
} = request;
let sniffed = mime::sniff(&data);
let source_format = source_image_format(sniffed.mime, content_type, filename);
let ImageTransformPlan {
format,
quality,
resize_mode,
response_content_type,
cache_key,
} = plan_image_transform(ImageTransformPlanRequest {
content_type,
cache_identity,
filename,
route,
source_format,
transform,
});
if same_format_loaded_image_request_can_use_original_with_sniff(
sniffed,
OriginalImageRequest {
source_ext: source_format,
explicit_out_ext: transform.explicit_format,
out_ext: format,
width: transform.width,
height: transform.height,
has_quality: transform.quality.is_some(),
effort: transform.effort,
animated: transform.animated,
},
) {
let serve_content_type = original_image_content_type(content_type, source_format);
return original_bytes_response(OriginalBytes {
method,
data,
content_type: &serve_content_type,
filename,
range_header,
requested_download: transform.requested_download,
});
}
let options = ImageOptions {
width: transform.width,
height: transform.height,
format,
quality,
animation: AnimationMode::new(transform.animated, runtime.animation()),
effort_override: transform.effort,
resize_mode,
deadline_ms: runtime.transform_deadline_ms(),
};
if let Some(hit) = cached_transform_hit(runtime, &cache_key) {
return transformed_response(TransformedBytes {
method,
data: hit.data,
content_type: response_content_type,
filename,
requested_download: transform.requested_download,
});
}
let can_serve_original = ((transform.width.is_none() && transform.height.is_none())
|| options.is_animated())
&& is_browser_displayable_image(sniffed.mime);
let fallback_original =
is_untransformed_fallback_image(sniffed.mime, source_format).then(|| data.clone());
let mut too_large_original = None;
let leader_original = &mut too_large_original;
let transformed = match cached_transform(CachedTransformRequest {
runtime,
cache_key,
format: options.format,
deadline: deadline_instant(options.deadline_ms),
work: move || async move {
let original = can_serve_original.then(|| data.clone());
let result = coalesced_work_result(run_transform(runtime, data, options).await);
if result
.as_ref()
.is_err_and(|error| error.downcast_ref() == Some(&MediaError::TooLargeToProcess))
{
*leader_original = original;
}
result.map(|media| MediaBytes::from(media.bytes))
},
})
.await
{
Ok(bytes) => bytes,
Err(error) => {
if let Some(original) = too_large_original {
return original_bytes_response(OriginalBytes {
method,
data: original,
content_type: sniffed.mime,
filename,
range_header,
requested_download: transform.requested_download,
});
}
if coalescer_error_is_transient(error)
&& let Some(original) = fallback_original
{
let mut response = original_bytes_response(OriginalBytes {
method,
data: original,
content_type: sniffed.mime,
filename,
range_header,
requested_download: transform.requested_download,
});
forbid_storing(&mut response);
return response;
}
if let Some(response) = coalescer_failure_response(error, "coalescer_timeout_image") {
return response;
}
return text_with_source(
StatusCode::BAD_REQUEST,
"Bad Request",
"image_transform_failed",
format!(
"route={:?} cache_identity={} fmt={} w={:?} h={:?} q={} animated={}",
route,
if route == TransformRoute::External {
crate::public_net_policy::external_url_for_log(cache_identity)
} else {
cache_identity.to_owned()
},
options.format.extension(),
options.width,
options.height,
options.quality,
transform.animated,
),
);
}
};
transformed_response(TransformedBytes {
method,
data: transformed,
content_type: response_content_type,
filename,
requested_download: transform.requested_download,
})
}
struct ImageTransformPlanRequest<'a> {
content_type: &'a str,
cache_identity: &'a str,
filename: &'a str,
route: TransformRoute,
source_format: Option<AssetExtension>,
transform: ValidatedTransformParameters,
}
struct ImageTransformPlan<'a> {
format: OutputFormat,
quality: ImageQuality,
resize_mode: ResizeMode,
response_content_type: &'a str,
cache_key: String,
}
fn plan_image_transform<'a>(request: ImageTransformPlanRequest<'a>) -> ImageTransformPlan<'a> {
let ImageTransformPlanRequest {
content_type,
cache_identity,
filename,
route,
source_format,
transform,
} = request;
let default_out_ext = match route {
TransformRoute::External => external_default_output_extension(filename, content_type),
TransformRoute::Attachment | TransformRoute::Stored | TransformRoute::Asset => {
image_extension_from_filename(filename).unwrap_or(AssetExtension::Webp)
}
};
let requested_format = transform.explicit_format.unwrap_or(default_out_ext);
let requested_supported_format = OutputFormat::coerce_from_extension(requested_format);
let format = effective_animated_image_output_format(
source_format,
requested_supported_format,
transform.animated,
);
let quality = transform.quality.unwrap_or_else(|| {
default_transform_quality(
format,
transform.animated,
transform_static_quality_default(source_format),
)
});
let resize_mode = if transform.wants_cover_crop() {
ResizeMode::Cover
} else {
ResizeMode::Fit
};
ImageTransformPlan {
format,
quality,
resize_mode,
response_content_type: transform_response_content_type(
transform.explicit_format,
requested_format,
format,
content_type,
),
cache_key: transform_cache_key(TransformCacheKeyInput {
route,
asset_kind: None,
cache_identity,
width: transform.width,
height: transform.height,
format,
quality: Some(quality),
animated: transform.animated,
effort: transform.effort,
resize_mode: Some(resize_mode),
}),
}
}
pub(in crate::server) struct CachedTransformProbe<'a> {
pub(in crate::server) method: Method,
pub(in crate::server) content_type: &'a str,
pub(in crate::server) source_format: Option<AssetExtension>,
pub(in crate::server) cache_identity: &'a str,
pub(in crate::server) filename: &'a str,
pub(in crate::server) route: TransformRoute,
pub(in crate::server) params: &'a HashMap<String, String>,
}
// Answers a repeat transform from the cache without the caller fetching the source
// bytes again, so a hot remote asset costs one upstream fetch per cache lifetime.
pub(in crate::server) fn cached_transform_response(
runtime: &TransformRuntime,
probe: CachedTransformProbe<'_>,
) -> Option<Response> {
let CachedTransformProbe {
method,
content_type,
source_format,
cache_identity,
filename,
route,
params,
} = probe;
let transform = validate_transform_parameters(params, &runtime.limits(), route).ok()?;
let source_is_svg = is_svg_content_type(content_type)
|| image_extension_from_filename(filename) == Some(AssetExtension::Svg);
if !source_is_svg && !transform.has_transform_request {
return None;
}
let (cache_key, response_content_type) = match mime::category(content_type)? {
mime::Category::Image => {
let plan = plan_image_transform(ImageTransformPlanRequest {
content_type,
cache_identity,
filename,
route,
source_format,
transform,
});
(plan.cache_key, plan.response_content_type.to_owned())
}
mime::Category::Video => {
let format = OutputFormat::coerce_from_extension(transform.explicit_format?);
let quality = transform.quality.unwrap_or(ImageQuality::High);
let cache_key = transform_cache_key(TransformCacheKeyInput {
route,
asset_kind: None,
cache_identity,
width: transform.width,
height: transform.height,
format,
quality: Some(quality),
animated: transform.animated,
effort: None,
resize_mode: None,
});
(cache_key, format.mime().to_owned())
}
mime::Category::Audio => return None,
};
let hit = cached_transform_hit(runtime, &cache_key)?;
Some(transformed_response(TransformedBytes {
method,
data: hit.data,
content_type: &response_content_type,
filename,
requested_download: transform.requested_download,
}))
}
struct TransformedBytes<'a> {
method: Method,
data: BudgetedBytes,
content_type: &'a str,
filename: &'a str,
requested_download: bool,
}
fn transformed_response(request: TransformedBytes<'_>) -> Response {
let disposition = Some(content_disposition_header(
request.content_type,
request.requested_download,
Some(request.filename),
));
generated_media_response(GeneratedMediaResponse {
method: request.method,
data: request.data,
content_type: request.content_type,
disposition,
})
}
pub(in crate::server) fn original_image_content_type(
content_type: &str,
source_format: Option<AssetExtension>,
) -> String {
if content_type.is_empty()
|| content_type.eq_ignore_ascii_case("application/octet-stream")
|| extension_from_mime(content_type).is_none()
{
return source_format
.map(|ext| ext.mime().to_owned())
.unwrap_or_else(|| content_type.to_owned());
}
content_type.to_owned()
}