Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(media-proxy): size own media metadata as served (#3621)

Hampusf315649e97c8
diff --git a/fluxer_media_proxy/src/server/external/mod.rs b/fluxer_media_proxy/src/server/external/mod.rs
index 6d3ead753..339d5849c 100644
--- a/fluxer_media_proxy/src/server/external/mod.rs
+++ b/fluxer_media_proxy/src/server/external/mod.rs
@@ -26,7 +26,7 @@ use crate::{
content_type_is_trustworthy, image_extension_from_filename, is_svg_content_type,
source_image_format,
},
- params::{animated_param, bool_param, url_filename},
+ params::{bool_param, requests_transform, url_filename},
response::{
content_disposition_header,
error::{text, text_with_source},
@@ -209,12 +209,7 @@ impl<'a> PreparedExternalRequest<'a> {
};
let url_ext_is_svg =
image_extension_from_filename(&url_filename(&url)) == Some(AssetExtension::Svg);
- let wants_transform = url_ext_is_svg
- || params.contains_key("width")
- || params.contains_key("height")
- || params.contains_key("format")
- || params.contains_key("quality")
- || animated_param(params, false);
+ let wants_transform = url_ext_is_svg || requests_transform(params);
let client_range = headers
.get(header::RANGE)
.and_then(|value| value.to_str().ok())
diff --git a/fluxer_media_proxy/src/server/media_operations/mod.rs b/fluxer_media_proxy/src/server/media_operations/mod.rs
index 416ba3274..6a6ab7b4a 100644
--- a/fluxer_media_proxy/src/server/media_operations/mod.rs
+++ b/fluxer_media_proxy/src/server/media_operations/mod.rs
@@ -19,13 +19,23 @@ use crate::{
nsfw::NSFWPolicy,
output_format::OutputFormat,
server::{
+ external::serve_external,
format_policy::image_extension_from_filename,
+ self_origin::{self, ServedRoute, ServedVariant},
state::AppState,
+ stored,
transform::execution::{deadline_instant, run_transform, transform_error_is_timeout},
},
};
+use axum::{
+ body::to_bytes,
+ http::{HeaderMap, Method, StatusCode},
+};
use bytes::Bytes;
use input::LoadedMediaInput;
+use std::{sync::Arc, time::Duration};
+
+const SERVED_VARIANT_TIMEOUT: Duration = Duration::from_secs(2);
pub(in crate::server) struct MetadataOutput {
pub(in crate::server) metadata: serde_json::Value,
@@ -33,11 +43,15 @@ pub(in crate::server) struct MetadataOutput {
}
pub(in crate::server) async fn resolve_metadata(
- app: &AppState,
+ app: &Arc<AppState>,
input: MediaInput,
scan_nsfw: bool,
include_data: bool,
) -> Result<MetadataOutput, MediaFailure> {
+ let served = match &input {
+ MediaInput::External { url, .. } if !include_data => self_origin::served_variant(app, url),
+ _ => None,
+ };
let mut input = load_media_input(app, input, MediaInputLimit::INTERNAL_REQUEST).await?;
if include_data && input.data.len() > constants::MAX_MEDIA_PROXY_BYTES {
return Err(MediaFailure::MetadataDecodedTooLarge);
@@ -51,9 +65,70 @@ pub(in crate::server) async fn resolve_metadata(
app.media.nsfw().record_declined_scan();
NSFWPolicy::Disabled
};
+ let mut metadata = extract_metadata(app, &input, true, nsfw).await?;
+ let is_image = metadata["content_type"]
+ .as_str()
+ .is_some_and(|content_type| content_type.starts_with("image/"));
+ if let Some(served) = served.filter(|_| is_image)
+ && let Some(mut served) = served_variant_metadata(app, served).await
+ {
+ for field in ["width", "height", "content_type", "format"] {
+ metadata[field] = served[field].take();
+ }
+ }
+ Ok(MetadataOutput {
+ metadata,
+ data: include_data.then_some(input.data),
+ })
+}
+
+async fn served_variant_metadata(
+ app: &Arc<AppState>,
+ served: ServedVariant,
+) -> Option<serde_json::Value> {
+ let ServedVariant { route, params } = served;
+ let headers = HeaderMap::new();
+ let work = async {
+ let response = match route {
+ ServedRoute::Asset(asset) => {
+ stored::serve_asset_image(app, Method::GET, asset, &params, &headers).await
+ }
+ ServedRoute::Attachment { key } => {
+ stored::serve_attachment(app, Method::GET, &key, &params, &headers).await
+ }
+ ServedRoute::External { rest } => {
+ serve_external(app, Method::GET, &rest, &params, &headers).await
+ }
+ };
+ if response.status() != StatusCode::OK {
+ return None;
+ }
+ let data = to_bytes(response.into_body(), constants::MAX_MEDIA_PROXY_BYTES)
+ .await
+ .ok()?;
+ let input = LoadedMediaInput {
+ data,
+ filename: "served".to_owned(),
+ };
+ extract_metadata(app, &input, false, NSFWPolicy::Disabled)
+ .await
+ .ok()
+ };
+ tokio::time::timeout(SERVED_VARIANT_TIMEOUT, work)
+ .await
+ .ok()
+ .flatten()
+}
+
+async fn extract_metadata(
+ app: &AppState,
+ input: &LoadedMediaInput,
+ placeholder: bool,
+ nsfw: NSFWPolicy,
+) -> Result<serde_json::Value, MediaFailure> {
let runtime = app.media.transforms();
let options = media_process::MetadataOptions {
- placeholder: true,
+ placeholder,
nsfw,
deadline_ms: runtime.transform_deadline_ms(),
};
@@ -84,10 +159,7 @@ pub(in crate::server) async fn resolve_metadata(
let json = media_process::finish_metadata(prepared, app.media.nsfw())
.await
.map_err(|err| extraction_failed(&err))?;
- Ok(MetadataOutput {
- metadata: serde_json::from_str(&json).unwrap_or_else(|_| serde_json::json!({})),
- data: include_data.then_some(input.data),
- })
+ Ok(serde_json::from_str(&json).unwrap_or_else(|_| serde_json::json!({})))
}
fn metadata_input_is_svg(input: &LoadedMediaInput) -> bool {
diff --git a/fluxer_media_proxy/src/server/params.rs b/fluxer_media_proxy/src/server/params.rs
index 1224dfa0c..4ab97c957 100644
--- a/fluxer_media_proxy/src/server/params.rs
+++ b/fluxer_media_proxy/src/server/params.rs
@@ -31,6 +31,14 @@ pub(in crate::server) fn animated_param(
.unwrap_or(default_value)
}
+pub(in crate::server) fn requests_transform(params: &HashMap<String, String>) -> bool {
+ params.contains_key("width")
+ || params.contains_key("height")
+ || params.contains_key("format")
+ || params.contains_key("quality")
+ || animated_param(params, false)
+}
+
pub(in crate::server) fn explicit_output_format(
params: &HashMap<String, String>,
) -> Result<Option<AssetExtension>, ()> {
diff --git a/fluxer_media_proxy/src/server/routes/dispatch.rs b/fluxer_media_proxy/src/server/routes/dispatch.rs
index a01d3e3b4..b546d760a 100644
--- a/fluxer_media_proxy/src/server/routes/dispatch.rs
+++ b/fluxer_media_proxy/src/server/routes/dispatch.rs
@@ -228,7 +228,9 @@ fn storage_key_decode_response(err: StorageKeyDecodeError) -> Response {
)
}
-fn without_signature_parameters(params: &HashMap<String, String>) -> HashMap<String, String> {
+pub(in crate::server) fn without_signature_parameters(
+ params: &HashMap<String, String>,
+) -> HashMap<String, String> {
params
.iter()
.filter(|(name, _)| !SIGNATURE_PARAMETER_NAMES.contains(&name.as_str()))
diff --git a/fluxer_media_proxy/src/server/routes/internal.rs b/fluxer_media_proxy/src/server/routes/internal.rs
index 91e154af2..1179baa30 100644
--- a/fluxer_media_proxy/src/server/routes/internal.rs
+++ b/fluxer_media_proxy/src/server/routes/internal.rs
@@ -539,6 +539,122 @@ mod tests {
String::from_utf8(bytes.to_vec()).expect("utf8 body")
}
+ #[tokio::test]
+ async fn own_url_metadata_reports_the_served_dimensions() {
+ let tmp = tempfile::tempdir().expect("temp storage root");
+ let storage_root = tmp.path().canonicalize().expect("canonical storage root");
+ let app = Arc::new(AppState::for_tests(
+ Config::load_from_iter([
+ (
+ "FLUXER_MEDIA_PROXY_SECRET_KEY".to_owned(),
+ "secret".to_owned(),
+ ),
+ (
+ "FLUXER_MEDIA_PROXY_STORAGE_ROOT".to_owned(),
+ storage_root.display().to_string(),
+ ),
+ (
+ "FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT".to_owned(),
+ "https://chat.example.test/media".to_owned(),
+ ),
+ ])
+ .expect("test config"),
+ ));
+ let cdn = &app.cfg.storage.bucket_cdn;
+ let emoji = crate::test_fixtures::synthetic_image(128, 128, ".webp[strip]");
+ for (key, data, content_type) in [
+ ("emojis/1", emoji.clone(), "image/webp"),
+ (
+ "emojis/2",
+ crate::test_fixtures::synthetic_image(128, 256, ".webp[strip,page-height=128]"),
+ "image/webp",
+ ),
+ (
+ "attachments/1/2/cat.png",
+ crate::test_fixtures::synthetic_png(128, 96),
+ "image/png",
+ ),
+ ] {
+ app.store
+ .write_object(cdn, key, &data, content_type)
+ .await
+ .expect("stored object");
+ }
+ let metadata = async |path: &str, with_base64: bool| {
+ let body = serde_json::json!({
+ "type": "external",
+ "url": format!("https://chat.example.test/media/{path}"),
+ "version": 2,
+ "nsfw": "allow",
+ "with_base64": with_base64,
+ });
+ let response = metadata_handler(
+ State(Arc::clone(&app)),
+ authorized_headers(),
+ json_request(body.to_string()),
+ )
+ .await;
+ assert_eq!(StatusCode::OK, response.status(), "{path}");
+ serde_json::from_str::<serde_json::Value>(&response_body(response).await)
+ .expect("metadata json")
+ };
+ for (path, width, height, content_type, animated) in [
+ ("emojis/1.webp?size=48", 48, 48, "image/webp", false),
+ ("emojis/1.webp?size=50", 56, 56, "image/webp", false),
+ ("emojis/1.webp?size=4096", 128, 128, "image/webp", false),
+ ("emojis/1.png?size=48", 48, 48, "image/png", false),
+ ("emojis/2.webp?size=48", 48, 48, "image/webp", true),
+ ("attachments/1/2/cat.png", 128, 96, "image/png", false),
+ (
+ "attachments/1/2/cat.png?width=64",
+ 64,
+ 48,
+ "image/png",
+ false,
+ ),
+ (
+ "attachments/1/2/cat.png?width=64&ex=0&is=0&hm=00&uc=dp",
+ 64,
+ 48,
+ "image/png",
+ false,
+ ),
+ (
+ "attachments/1/2/cat.png?width=nope",
+ 128,
+ 96,
+ "image/png",
+ false,
+ ),
+ ] {
+ let metadata = metadata(path, false).await;
+ assert_eq!(
+ (
+ Some(width),
+ Some(height),
+ Some(content_type),
+ Some(animated)
+ ),
+ (
+ metadata["width"].as_u64(),
+ metadata["height"].as_u64(),
+ metadata["content_type"].as_str(),
+ metadata["animated"].as_bool(),
+ ),
+ "{path}"
+ );
+ }
+ let metadata = metadata("emojis/1.webp?size=48", true).await;
+ assert_eq!(
+ (Some(128), Some(128)),
+ (metadata["width"].as_u64(), metadata["height"].as_u64())
+ );
+ assert_eq!(
+ Some(general_purpose::STANDARD.encode(&emoji).as_str()),
+ metadata["base64"].as_str()
+ );
+ }
+
#[test]
fn internal_auth_uses_bearer_secret() {
let headers = authorized_headers();
diff --git a/fluxer_media_proxy/src/server/self_origin.rs b/fluxer_media_proxy/src/server/self_origin.rs
index 9066f3096..d5c9a763c 100644
--- a/fluxer_media_proxy/src/server/self_origin.rs
+++ b/fluxer_media_proxy/src/server/self_origin.rs
@@ -6,12 +6,15 @@ use crate::{
public_net_policy,
server::{
asset_path::{
- decode_storage_key, parse_entrance_sound_path, parse_guild_member_asset_path,
- parse_simple_asset_path, parse_standard_asset_path,
+ ParsedAssetPath, decode_storage_key, parse_entrance_sound_path,
+ parse_guild_member_asset_path, parse_simple_asset_path, parse_standard_asset_path,
},
+ params::requests_transform,
+ routes::dispatch::without_signature_parameters,
state::AppState,
},
};
+use std::collections::HashMap;
pub(in crate::server) enum SelfOrigin {
Stored {
@@ -24,9 +27,20 @@ pub(in crate::server) enum SelfOrigin {
},
}
+pub(in crate::server) struct ServedVariant {
+ pub(in crate::server) route: ServedRoute,
+ pub(in crate::server) params: HashMap<String, String>,
+}
+
+pub(in crate::server) enum ServedRoute {
+ Asset(ParsedAssetPath),
+ Attachment { key: String },
+ External { rest: String },
+}
+
pub(in crate::server) fn resolve(app: &AppState, url: &str) -> Option<SelfOrigin> {
let endpoint = app.cfg.public_endpoint.as_deref()?;
- let path = own_path(endpoint, url)?;
+ let (path, _) = own_path(endpoint, url)?;
let cdn = &app.cfg.storage.bucket_cdn;
if app.cfg.mode == DeploymentMode::Static {
return stored(
@@ -59,10 +73,7 @@ pub(in crate::server) fn resolve(app: &AppState, url: &str) -> Option<SelfOrigin
if let Some(key) = parse_entrance_sound_path(path) {
return stored(cdn, key);
}
- let asset = parse_guild_member_asset_path(path)
- .or_else(|| parse_simple_asset_path(path, AssetKind::Emoji))
- .or_else(|| parse_simple_asset_path(path, AssetKind::Sticker))
- .or_else(|| parse_standard_asset_path(path))?;
+ let asset = parse_asset_path(path)?;
Some(SelfOrigin::Stored {
bucket: cdn.clone(),
key: asset.storage_key,
@@ -70,6 +81,47 @@ pub(in crate::server) fn resolve(app: &AppState, url: &str) -> Option<SelfOrigin
})
}
+pub(in crate::server) fn served_variant(app: &AppState, url: &str) -> Option<ServedVariant> {
+ let endpoint = app.cfg.public_endpoint.as_deref()?;
+ let (path, query) = own_path(endpoint, url)?;
+ if app.cfg.mode == DeploymentMode::Static {
+ return None;
+ }
+ let params: HashMap<String, String> = url::form_urlencoded::parse(query.as_bytes())
+ .into_owned()
+ .collect();
+ if let Some(rest) = path.strip_prefix("/external/") {
+ return requests_transform(&params).then(|| ServedVariant {
+ route: ServedRoute::External {
+ rest: rest.to_owned(),
+ },
+ params,
+ });
+ }
+ if path.starts_with("/attachments/") {
+ let params = without_signature_parameters(&params);
+ let key = decode_storage_key(path).ok()?;
+ return requests_transform(&params).then_some(ServedVariant {
+ route: ServedRoute::Attachment { key },
+ params,
+ });
+ }
+ if path.starts_with("/themes/") || parse_entrance_sound_path(path).is_some() {
+ return None;
+ }
+ Some(ServedVariant {
+ route: ServedRoute::Asset(parse_asset_path(path)?),
+ params,
+ })
+}
+
+fn parse_asset_path(path: &str) -> Option<ParsedAssetPath> {
+ parse_guild_member_asset_path(path)
+ .or_else(|| parse_simple_asset_path(path, AssetKind::Emoji))
+ .or_else(|| parse_simple_asset_path(path, AssetKind::Sticker))
+ .or_else(|| parse_standard_asset_path(path))
+}
+
fn stored(bucket: &str, key: String) -> Option<SelfOrigin> {
Some(SelfOrigin::Stored {
bucket: bucket.to_owned(),
@@ -78,7 +130,7 @@ fn stored(bucket: &str, key: String) -> Option<SelfOrigin> {
})
}
-fn own_path<'a>(endpoint: &str, url: &'a str) -> Option<&'a str> {
+fn own_path<'a>(endpoint: &str, url: &'a str) -> Option<(&'a str, &'a str)> {
let base = public_net_policy::parse_url(endpoint).ok()?;
let target = public_net_policy::parse_url(url).ok()?;
if !base.scheme.eq_ignore_ascii_case(target.scheme) {
@@ -90,16 +142,14 @@ fn own_path<'a>(endpoint: &str, url: &'a str) -> Option<&'a str> {
if effective_port(base.scheme, base.port) != effective_port(target.scheme, target.port) {
return None;
}
- let prefix = path_of(base.path_query).trim_end_matches('/');
- let rest = path_of(target.path_query).strip_prefix(prefix)?;
- rest.starts_with('/').then_some(rest)
+ let prefix = split_query(base.path_query).0.trim_end_matches('/');
+ let (path, query) = split_query(target.path_query);
+ let rest = path.strip_prefix(prefix)?;
+ rest.starts_with('/').then_some((rest, query))
}
-fn path_of(path_query: &str) -> &str {
- path_query
- .split_once('?')
- .map(|(path, _)| path)
- .unwrap_or(path_query)
+fn split_query(path_query: &str) -> (&str, &str) {
+ path_query.split_once('?').unwrap_or((path_query, ""))
}
fn effective_port(scheme: &str, port: Option<u16>) -> u16 {
diff --git a/fluxer_media_proxy/src/server/stored/mod.rs b/fluxer_media_proxy/src/server/stored/mod.rs
index a42789085..d65ced803 100644
--- a/fluxer_media_proxy/src/server/stored/mod.rs
+++ b/fluxer_media_proxy/src/server/stored/mod.rs
@@ -21,8 +21,8 @@ use crate::{
},
media_operations::spool_stored_media,
params::{
- animated_param, asset_manual_format_override, asset_wants_animated, bool_param,
- filename_from_storage_key,
+ asset_manual_format_override, asset_wants_animated, bool_param,
+ filename_from_storage_key, requests_transform,
},
response::{
GeneratedMediaResponse, MediaResponse, content_disposition_header,
@@ -323,13 +323,7 @@ async fn read_attachment(
headers: &HeaderMap,
) -> Response {
let filename = filename_from_storage_key(key);
- let animated = animated_param(params, false);
- let wants_transform = params.contains_key("width")
- || params.contains_key("height")
- || params.contains_key("format")
- || params.contains_key("quality")
- || animated;
- if !wants_transform {
+ if !requests_transform(params) {
return serve_stored_passthrough_stream(
app,
method,
diff --git a/fluxer_media_proxy/src/server/transform/parameters.rs b/fluxer_media_proxy/src/server/transform/parameters.rs
index 2f7f4195d..627c01561 100644
--- a/fluxer_media_proxy/src/server/transform/parameters.rs
+++ b/fluxer_media_proxy/src/server/transform/parameters.rs
@@ -8,7 +8,7 @@ use crate::{
server::{
params::{
animated_param, bool_param, explicit_output_format, parse_effort,
- parse_optional_dimension_param,
+ parse_optional_dimension_param, requests_transform,
},
response::error::text_with_reason,
},
@@ -68,11 +68,7 @@ pub(in crate::server) fn validate_transform_parameters(
animated,
effort,
requested_download: bool_param(params, "download", false),
- has_transform_request: params.contains_key("width")
- || params.contains_key("height")
- || params.contains_key("format")
- || params.contains_key("quality")
- || animated,
+ has_transform_request: requests_transform(params),
})
}