A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(media-proxy): retry relay uploads on dropped connections (#2751)
Hampusa9cc04d2772a
diff --git a/fluxer_media_proxy/src/http_client/mod.rs b/fluxer_media_proxy/src/http_client/mod.rsindex e04118e5e..cab47acd7 100644--- a/fluxer_media_proxy/src/http_client/mod.rs+++ b/fluxer_media_proxy/src/http_client/mod.rs@@ -6,6 +6,7 @@ use crate::metrics::http_client::HTTPClientMetrics;use crate::public_net_policy::PinnedDnsResolver;use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff};+pub(crate) use retry::is_dropped_connection;use retry::{MediaProxyRetryStrategy, ObservableRetryPolicy};use std::num::{NonZeroU32, NonZeroU64};use std::sync::Arc;diff --git a/fluxer_media_proxy/src/http_client/retry.rs b/fluxer_media_proxy/src/http_client/retry.rsindex 490c0d541..37f139ba7 100644--- a/fluxer_media_proxy/src/http_client/retry.rs+++ b/fluxer_media_proxy/src/http_client/retry.rs@@ -104,7 +104,7 @@ fn retryable_error(error: &MiddlewareError) -> Option<Retryable> {}}-fn is_dropped_connection(error: &(dyn std::error::Error + 'static)) -> bool {+pub(crate) fn is_dropped_connection(error: &(dyn std::error::Error + 'static)) -> bool {let mut current = Some(error);while let Some(error) = current {if errordiff --git a/fluxer_media_proxy/src/server/relay/body.rs b/fluxer_media_proxy/src/server/relay/body.rsindex bcd8abf00..cc4be1383 100644--- a/fluxer_media_proxy/src/server/relay/body.rs+++ b/fluxer_media_proxy/src/server/relay/body.rs@@ -2,7 +2,8 @@use crate::{storage::RelayBodyChunks, upload_relay::RelayError};use axum::{body::Body, http::HeaderValue};-use bytes::Bytes;+use bytes::{Bytes, BytesMut};+use futures_util::StreamExt as _;use http_body_util::BodyExt as _;use parking_lot::Mutex;use std::{@@ -221,6 +222,17 @@ impl RelayBodyActiveStream {}}+pub(in crate::server) async fn buffer_relay_body(+ mut chunks: RelayBodyChunks,+ declared_length: u64,+) -> Result<Bytes, io::Error> {+ let mut buffered = BytesMut::with_capacity(usize::try_from(declared_length).unwrap_or(0));+ while let Some(chunk) = chunks.next().await {+ buffered.extend_from_slice(&chunk?);+ }+ Ok(buffered.freeze())+}+pub(in crate::server) fn relay_body_chunks(stream: RelayBodyStream) -> RelayBodyChunks {Box::pin(futures_util::stream::unfold(Some(stream),diff --git a/fluxer_media_proxy/src/server/relay/tests.rs b/fluxer_media_proxy/src/server/relay/tests.rsindex 1f27e8709..0639550d2 100644--- a/fluxer_media_proxy/src/server/relay/tests.rs+++ b/fluxer_media_proxy/src/server/relay/tests.rs@@ -24,7 +24,7 @@ use base64::engine::general_purpose::STANDARD;use bytes::Bytes;use std::{collections::HashMap,- sync::{Arc, OnceLock},+ sync::{Arc, OnceLock, atomic::Ordering},time::Duration,};@@ -592,6 +592,118 @@ fn relay_etag_degrades_an_unrepresentable_upstream_value_to_an_empty_header() {assert_eq!("\"etag-123\"", relay_etag("\"etag-123\"").to_str().unwrap());}+struct DroppedConnectionRelay {+ status: StatusCode,+ accepted: usize,+ stored_puts: Vec<Bytes>,+ metrics: String,+}++async fn relay_put_through_a_dropped_connection(+ body: Bytes,+ declared_length: Option<u64>,+ buffered_retry_max_bytes: u64,+) -> DroppedConnectionRelay {+ let fake = crate::storage::tests::fake_s3().await;+ let (endpoint, accepted) =+ crate::storage::tests::connection_dropping_front(fake.endpoint(), 1).await;+ let tmp = tempfile::tempdir().unwrap();+ let relay_secret = [7u8; 32];+ let mut cfg = fake.config(tmp.path());+ cfg.storage.s3_endpoint = endpoint;+ cfg.mode = crate::config::DeploymentMode::Upload;+ cfg.socket_io_timeout_ms = 30_000;+ cfg.upload_relay.secret = SecretBytes::new(relay_secret.to_vec());+ cfg.upload_relay.max_body_bytes = 4096;+ cfg.upload_relay.buffered_retry_max_bytes = buffered_retry_max_bytes;+ cfg.upload_relay.buffered_retry_total_bytes = 1 << 20;+ let key = "guild/dropped.bin";+ let token = relay_test_token(key, &relay_secret);+ let app = test_app_state(cfg);++ let response = relay_put(+ State(Arc::clone(&app)),+ Path(key.to_owned()),+ Query(HashMap::from([("t".to_owned(), token)])),+ declared_length+ .map(content_length_headers)+ .unwrap_or_default(),+ Request::builder()+ .method(Method::PUT)+ .body(Body::from(body))+ .unwrap(),+ )+ .await;++ DroppedConnectionRelay {+ status: response.status(),+ accepted: accepted.load(Ordering::SeqCst),+ stored_puts: fake+ .requests()+ .into_iter()+ .filter(|(method, ..)| *method == Method::PUT)+ .map(|(.., sent)| sent)+ .collect(),+ metrics: app.metrics.render(),+ }+}++#[tokio::test]+async fn relay_put_retries_a_buffered_body_after_a_dropped_connection() {+ let body = Bytes::from_static(b"buffered upload bytes");++ let relay =+ relay_put_through_a_dropped_connection(body.clone(), Some(body.len() as u64), 4096).await;++ assert_eq!(StatusCode::OK, relay.status);+ assert_eq!(2, relay.accepted);+ assert_eq!(vec![body], relay.stored_puts);+ assert!(+ relay+ .metrics+ .contains("fluxer_media_proxy_relay_upstream_retries_total 1"),+ "{}",+ relay.metrics+ );+}++#[tokio::test]+async fn relay_put_retries_a_spooled_body_after_a_dropped_connection() {+ let body = Bytes::from_static(b"spooled upload bytes");++ let relay = relay_put_through_a_dropped_connection(body.clone(), None, 4096).await;++ assert_eq!(StatusCode::OK, relay.status);+ assert_eq!(2, relay.accepted);+ assert_eq!(vec![body], relay.stored_puts);+ assert!(+ relay+ .metrics+ .contains("fluxer_media_proxy_relay_upstream_retries_total 1"),+ "{}",+ relay.metrics+ );+}++#[tokio::test]+async fn relay_put_does_not_retry_a_body_too_large_to_buffer() {+ let body = Bytes::from_static(b"streamed upload bytes");++ let relay =+ relay_put_through_a_dropped_connection(body.clone(), Some(body.len() as u64), 4).await;++ assert_eq!(StatusCode::BAD_GATEWAY, relay.status);+ assert_eq!(1, relay.accepted);+ assert!(relay.stored_puts.is_empty());+ assert!(+ relay+ .metrics+ .contains("fluxer_media_proxy_relay_upstream_retries_total 0"),+ "{}",+ relay.metrics+ );+}+#[tokio::test]async fn relay_put_omits_the_etag_when_the_store_returns_none() {let tmp = tempfile::tempdir().unwrap();diff --git a/fluxer_media_proxy/src/server/routes/relay.rs b/fluxer_media_proxy/src/server/routes/relay.rsindex 9c5394ddf..77306554b 100644--- a/fluxer_media_proxy/src/server/routes/relay.rs+++ b/fluxer_media_proxy/src/server/routes/relay.rs@@ -2,18 +2,18 @@use crate::{config::DeploymentMode,- external_path, http_headers,+ external_path, http_client, http_headers,server::{relay::body::{- RelayBodyProgress, RelayBodyStream, RelayBodyStreamRequest, relay_body_chunks,- relay_etag, validate_completed_relay_body,+ RelayBodyProgress, RelayBodyStream, RelayBodyStreamRequest, buffer_relay_body,+ relay_body_chunks, relay_etag, validate_completed_relay_body,},response::error::text,state::AppState,},spool::{SpoolError, spool_to_temp},storage::{RelayBody, RelayPutOptions},- upload_relay::{RelayError, target, token},+ upload_relay::{BufferReservation, RelayError, target, token},};use axum::{body::Body,@@ -91,7 +91,7 @@ pub(in crate::server) async fn relay_put(return relay_error(err);}let timeout_ms = relay_upstream_timeout_ms(&app, content_length);- let (body, body_length, streamed) = match content_length {+ let (body, body_length, streamed, _buffer_reservation) = match content_length {Some(declared) => {let Some(deadline) =tokio::time::Instant::now().checked_add(Duration::from_millis(timeout_ms))@@ -106,11 +106,34 @@ pub(in crate::server) async fn relay_put(failure: Arc::clone(&failure),});let progress = stream.progress();- (- RelayBody::Streamed(relay_body_chunks(stream)),- declared,- Some(StreamedRelayBody { failure, progress }),- )+ match buffer_reservation(&app, declared) {+ Some(reservation) => {+ let Ok(buffered) = buffer_relay_body(relay_body_chunks(stream), declared).await+ else {+ return relay_error(+ failure+ .get()+ .copied()+ .unwrap_or(RelayError::ClientUploadFailed),+ );+ };+ if let Err(err) = validate_completed_relay_body(&failure, &progress, declared) {+ return relay_error(err);+ }+ (+ RelayBody::Buffered(buffered),+ declared,+ None,+ Some(reservation),+ )+ }+ None => (+ RelayBody::Streamed(relay_body_chunks(stream)),+ declared,+ Some(StreamedRelayBody { failure, progress }),+ None,+ ),+ }}None => {let body_length_limit = token.mb.min(app.cfg.upload_relay.max_body_bytes);@@ -140,7 +163,7 @@ pub(in crate::server) async fn relay_put(}};let (file, spooled_length) = spooled.into_parts();- (RelayBody::Spooled(file), spooled_length, None)+ (RelayBody::Spooled(file), spooled_length, None, None)}};let content_type = resolve_relay_content_type(@@ -149,19 +172,32 @@ pub(in crate::server) async fn relay_put(.get(header::CONTENT_TYPE).and_then(|v| v.to_str().ok()),);- let options = RelayPutOptions {+ let upload_id = params.get("uploadId").cloned();+ let relay_options = |body| RelayPutOptions {body,content_length: body_length,- content_type,- upload_id: params.get("uploadId").cloned(),+ content_type: content_type.clone(),+ upload_id: upload_id.clone(),part_number,timeout_ms,};- match app+ let (body, replay) = with_replay(body).await;+ let mut outcome = app.store- .relay_put_object(&app.cfg.storage.bucket_uploads, &key, options)- .await- {+ .relay_put_object(&app.cfg.storage.bucket_uploads, &key, relay_options(body))+ .await;+ let retry_body = match &outcome {+ Err(err) if http_client::is_dropped_connection(err) => replay,+ _ => None,+ };+ if let Some(body) = retry_body {+ app.metrics.relay().record_retry();+ outcome = app+ .store+ .relay_put_object(&app.cfg.storage.bucket_uploads, &key, relay_options(body))+ .await;+ }+ match outcome {Ok(etag) => match relay_success_etag(etag, streamed.as_ref(), body_length) {Ok(etag) => {app.metrics.relay().record_success();@@ -183,6 +219,30 @@ pub(in crate::server) async fn relay_put(}}+fn buffer_reservation(app: &AppState, declared_length: u64) -> Option<BufferReservation> {+ if declared_length > app.cfg.upload_relay.buffered_retry_max_bytes {+ return None;+ }+ BufferReservation::try_new(+ declared_length,+ app.cfg.upload_relay.buffered_retry_total_bytes,+ )+}++async fn with_replay(body: RelayBody) -> (RelayBody, Option<RelayBody>) {+ match body {+ RelayBody::Buffered(bytes) => (+ RelayBody::Buffered(bytes.clone()),+ Some(RelayBody::Buffered(bytes)),+ ),+ RelayBody::Spooled(file) => {+ let replay = file.try_clone().await.ok().map(RelayBody::Spooled);+ (RelayBody::Spooled(file), replay)+ }+ RelayBody::Streamed(chunks) => (RelayBody::Streamed(chunks), None),+ }+}+fn relay_upstream_timeout_ms(app: &AppState, content_length: Option<u64>) -> u64 {let Some(declared) = content_length else {return app.cfg.upload_relay.s3_timeout_ms;diff --git a/fluxer_media_proxy/src/storage/local.rs b/fluxer_media_proxy/src/storage/local.rsindex 8f86b8a46..e08c8fb03 100644--- a/fluxer_media_proxy/src/storage/local.rs+++ b/fluxer_media_proxy/src/storage/local.rs@@ -202,6 +202,9 @@ impl Store {dest.write_all(&chunk?).await?;}}+ RelayBody::Buffered(bytes) => {+ dest.write_all(&bytes).await?;+ }}dest.flush().await?;dest.sync_all().await?;diff --git a/fluxer_media_proxy/src/storage/relay_body.rs b/fluxer_media_proxy/src/storage/relay_body.rsindex fd34690d2..a4d35f518 100644--- a/fluxer_media_proxy/src/storage/relay_body.rs+++ b/fluxer_media_proxy/src/storage/relay_body.rs@@ -17,6 +17,7 @@ pub type RelayBodyChunks =pub enum RelayBody {Spooled(tokio::fs::File),Streamed(RelayBodyChunks),+ Buffered(Bytes),}pub struct RelayPutOptions {diff --git a/fluxer_media_proxy/src/storage/s3.rs b/fluxer_media_proxy/src/storage/s3.rsindex df4e8ac02..8a9f1b8eb 100644--- a/fluxer_media_proxy/src/storage/s3.rs+++ b/fluxer_media_proxy/src/storage/s3.rs@@ -339,6 +339,7 @@ impl Store {reqwest::Body::wrap(SizedFileBody::new(file, options.content_length))}RelayBody::Streamed(chunks) => reqwest::Body::wrap_stream(chunks),+ RelayBody::Buffered(bytes) => reqwest::Body::from(bytes),};let response = self.raw_clientdiff --git a/fluxer_media_proxy/src/storage/tests/mod.rs b/fluxer_media_proxy/src/storage/tests/mod.rsindex b9b1e9aa2..82a085d31 100644--- a/fluxer_media_proxy/src/storage/tests/mod.rs+++ b/fluxer_media_proxy/src/storage/tests/mod.rs@@ -21,7 +21,16 @@ use axum::response::Response;use bytes::Bytes;use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, header};use parking_lot::Mutex;-use std::{collections::HashMap, path::Path, sync::Arc, time::Duration};+use std::{+ collections::HashMap,+ path::Path,+ sync::{+ Arc,+ atomic::{AtomicUsize, Ordering},+ },+ time::Duration,+};+use tokio::io::AsyncReadExt as _;pub(crate) type CapturedRequest = (Method, http::Uri, HeaderMap, Bytes);@@ -287,3 +296,30 @@ fn fake_s3_response(}response}++pub(crate) async fn connection_dropping_front(+ origin: &str,+ dropped: usize,+) -> (String, Arc<AtomicUsize>) {+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let endpoint = format!("http://{}", listener.local_addr().unwrap());+ let origin = origin.trim_start_matches("http://").to_owned();+ let accepted = Arc::new(AtomicUsize::new(0));+ let counter = Arc::clone(&accepted);+ tokio::spawn(async move {+ while let Ok((mut client, _)) = listener.accept().await {+ let index = counter.fetch_add(1, Ordering::SeqCst);+ let origin = origin.clone();+ tokio::spawn(async move {+ if index < dropped {+ let mut request = [0u8; 4096];+ let _ = client.read(&mut request).await;+ return;+ }+ let mut upstream = tokio::net::TcpStream::connect(origin).await.unwrap();+ let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await;+ });+ }+ });+ (endpoint, accepted)+}diff --git a/fluxer_media_proxy/src/storage/tests/s3.rs b/fluxer_media_proxy/src/storage/tests/s3.rsindex 14372a003..86d122a6a 100644--- a/fluxer_media_proxy/src/storage/tests/s3.rs+++ b/fluxer_media_proxy/src/storage/tests/s3.rs@@ -1,6 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use super::{FakeObject, FakeS3, fake_s3, store};+use super::{FakeObject, FakeS3, connection_dropping_front, fake_s3, store};use crate::{byte_budget::ByteBudget,range::ByteRange,@@ -8,14 +8,7 @@ use crate::{storage::{ObjectReadRequest, ObjectStreamRequest, StorageError},};use http::{Method, StatusCode, header};-use std::{- sync::{- Arc,- atomic::{AtomicUsize, Ordering},- },- time::Duration,-};-use tokio::io::AsyncReadExt as _;+use std::{sync::atomic::Ordering, time::Duration};const LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT";@@ -393,30 +386,6 @@ async fn a_read_that_keeps_losing_its_connection_reports_the_cause() {assert!(fake.requests().is_empty());}-async fn connection_dropping_front(origin: &str, dropped: usize) -> (String, Arc<AtomicUsize>) {- let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();- let endpoint = format!("http://{}", listener.local_addr().unwrap());- let origin = origin.trim_start_matches("http://").to_owned();- let accepted = Arc::new(AtomicUsize::new(0));- let counter = Arc::clone(&accepted);- tokio::spawn(async move {- while let Ok((mut client, _)) = listener.accept().await {- let index = counter.fetch_add(1, Ordering::SeqCst);- let origin = origin.clone();- tokio::spawn(async move {- if index < dropped {- let mut request = [0u8; 4096];- let _ = client.read(&mut request).await;- return;- }- let mut upstream = tokio::net::TcpStream::connect(origin).await.unwrap();- let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await;- });- }- });- (endpoint, accepted)-}-fn fake_gets(fake: &FakeS3, path: &str) -> usize {fake.requests().iter()diff --git a/fluxer_media_proxy/src/upload_relay/mod.rs b/fluxer_media_proxy/src/upload_relay/mod.rsindex a4ba2ad79..a31ac1b63 100644--- a/fluxer_media_proxy/src/upload_relay/mod.rs+++ b/fluxer_media_proxy/src/upload_relay/mod.rs@@ -53,6 +53,22 @@ pub fn release_buffer_budget(amount: u64) {BUFFERED_RETRY_IN_FLIGHT.fetch_sub(amount, Ordering::AcqRel);}+pub struct BufferReservation {+ bytes: u64,+}++impl BufferReservation {+ pub fn try_new(bytes: u64, ceiling: u64) -> Option<Self> {+ try_reserve_buffer_budget(bytes, ceiling).then(|| Self { bytes })+ }+}++impl Drop for BufferReservation {+ fn drop(&mut self) {+ release_buffer_budget(self.bytes);+ }+}+pub fn try_reserve_spool_budget(needed: u64, ceiling: u64) -> bool {try_reserve(&SPOOL_IN_FLIGHT_BYTES, needed, ceiling)}diff --git a/fluxer_media_proxy/src/upload_relay/tests.rs b/fluxer_media_proxy/src/upload_relay/tests.rsindex c037bb3e3..d0bf6f0b6 100644--- a/fluxer_media_proxy/src/upload_relay/tests.rs+++ b/fluxer_media_proxy/src/upload_relay/tests.rs@@ -1,12 +1,13 @@// SPDX-License-Identifier: AGPL-3.0-or-lateruse crate::upload_relay::{- RelayError, release_buffer_budget,+ RelayError,target::{RelayRequest, query_part_number, valid_content_type, validate_relay_request},token::{TokenError, TokenMethod, TokenPayload, decode_token, encode_token, token_from_query},- try_reserve_buffer_budget,+ try_reserve,};use http::Method;+use std::sync::atomic::{AtomicU64, Ordering};const BASE64_URL_ALPHABET: &[u8] =b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";@@ -124,11 +125,12 @@ fn validates_relay_mismatches() {#[test]fn buffer_budget_is_bounded() {- assert!(try_reserve_buffer_budget(4, 8));- assert!(!try_reserve_buffer_budget(5, 8));- release_buffer_budget(4);- assert!(try_reserve_buffer_budget(8, 8));- release_buffer_budget(8);+ let in_flight = AtomicU64::new(0);+ assert!(try_reserve(&in_flight, 4, 8));+ assert!(!try_reserve(&in_flight, 5, 8));+ in_flight.fetch_sub(4, Ordering::AcqRel);+ assert!(try_reserve(&in_flight, 8, 8));+ assert!(!try_reserve(&AtomicU64::new(0), 1, 0));}#[test]