Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(media-proxy): retry relay uploads on dropped connections (#2751)

Hampusa9cc04d2772a
diff --git a/fluxer_media_proxy/src/http_client/mod.rs b/fluxer_media_proxy/src/http_client/mod.rs
index e04118e5e..cab47acd7 100644
--- a/fluxer_media_proxy/src/http_client/mod.rs
+++ b/fluxer_media_proxy/src/http_client/mod.rs
@@ -6,6 +6,7 @@ use crate::metrics::http_client::HTTPClientMetrics;
use crate::public_net_policy::PinnedDnsResolver;
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff};
+pub(crate) use retry::is_dropped_connection;
use retry::{MediaProxyRetryStrategy, ObservableRetryPolicy};
use std::num::{NonZeroU32, NonZeroU64};
use std::sync::Arc;
diff --git a/fluxer_media_proxy/src/http_client/retry.rs b/fluxer_media_proxy/src/http_client/retry.rs
index 490c0d541..37f139ba7 100644
--- a/fluxer_media_proxy/src/http_client/retry.rs
+++ b/fluxer_media_proxy/src/http_client/retry.rs
@@ -104,7 +104,7 @@ fn retryable_error(error: &MiddlewareError) -> Option<Retryable> {
}
}
-fn is_dropped_connection(error: &(dyn std::error::Error + 'static)) -> bool {
+pub(crate) fn is_dropped_connection(error: &(dyn std::error::Error + 'static)) -> bool {
let mut current = Some(error);
while let Some(error) = current {
if error
diff --git a/fluxer_media_proxy/src/server/relay/body.rs b/fluxer_media_proxy/src/server/relay/body.rs
index bcd8abf00..cc4be1383 100644
--- a/fluxer_media_proxy/src/server/relay/body.rs
+++ b/fluxer_media_proxy/src/server/relay/body.rs
@@ -2,7 +2,8 @@
use crate::{storage::RelayBodyChunks, upload_relay::RelayError};
use axum::{body::Body, http::HeaderValue};
-use bytes::Bytes;
+use bytes::{Bytes, BytesMut};
+use futures_util::StreamExt as _;
use http_body_util::BodyExt as _;
use parking_lot::Mutex;
use std::{
@@ -221,6 +222,17 @@ impl RelayBodyActiveStream {
}
}
+pub(in crate::server) async fn buffer_relay_body(
+ mut chunks: RelayBodyChunks,
+ declared_length: u64,
+) -> Result<Bytes, io::Error> {
+ let mut buffered = BytesMut::with_capacity(usize::try_from(declared_length).unwrap_or(0));
+ while let Some(chunk) = chunks.next().await {
+ buffered.extend_from_slice(&chunk?);
+ }
+ Ok(buffered.freeze())
+}
+
pub(in crate::server) fn relay_body_chunks(stream: RelayBodyStream) -> RelayBodyChunks {
Box::pin(futures_util::stream::unfold(
Some(stream),
diff --git a/fluxer_media_proxy/src/server/relay/tests.rs b/fluxer_media_proxy/src/server/relay/tests.rs
index 1f27e8709..0639550d2 100644
--- a/fluxer_media_proxy/src/server/relay/tests.rs
+++ b/fluxer_media_proxy/src/server/relay/tests.rs
@@ -24,7 +24,7 @@ use base64::engine::general_purpose::STANDARD;
use bytes::Bytes;
use std::{
collections::HashMap,
- sync::{Arc, OnceLock},
+ sync::{Arc, OnceLock, atomic::Ordering},
time::Duration,
};
@@ -592,6 +592,118 @@ fn relay_etag_degrades_an_unrepresentable_upstream_value_to_an_empty_header() {
assert_eq!("\"etag-123\"", relay_etag("\"etag-123\"").to_str().unwrap());
}
+struct DroppedConnectionRelay {
+ status: StatusCode,
+ accepted: usize,
+ stored_puts: Vec<Bytes>,
+ metrics: String,
+}
+
+async fn relay_put_through_a_dropped_connection(
+ body: Bytes,
+ declared_length: Option<u64>,
+ buffered_retry_max_bytes: u64,
+) -> DroppedConnectionRelay {
+ let fake = crate::storage::tests::fake_s3().await;
+ let (endpoint, accepted) =
+ crate::storage::tests::connection_dropping_front(fake.endpoint(), 1).await;
+ let tmp = tempfile::tempdir().unwrap();
+ let relay_secret = [7u8; 32];
+ let mut cfg = fake.config(tmp.path());
+ cfg.storage.s3_endpoint = endpoint;
+ cfg.mode = crate::config::DeploymentMode::Upload;
+ cfg.socket_io_timeout_ms = 30_000;
+ cfg.upload_relay.secret = SecretBytes::new(relay_secret.to_vec());
+ cfg.upload_relay.max_body_bytes = 4096;
+ cfg.upload_relay.buffered_retry_max_bytes = buffered_retry_max_bytes;
+ cfg.upload_relay.buffered_retry_total_bytes = 1 << 20;
+ let key = "guild/dropped.bin";
+ let token = relay_test_token(key, &relay_secret);
+ let app = test_app_state(cfg);
+
+ let response = relay_put(
+ State(Arc::clone(&app)),
+ Path(key.to_owned()),
+ Query(HashMap::from([("t".to_owned(), token)])),
+ declared_length
+ .map(content_length_headers)
+ .unwrap_or_default(),
+ Request::builder()
+ .method(Method::PUT)
+ .body(Body::from(body))
+ .unwrap(),
+ )
+ .await;
+
+ DroppedConnectionRelay {
+ status: response.status(),
+ accepted: accepted.load(Ordering::SeqCst),
+ stored_puts: fake
+ .requests()
+ .into_iter()
+ .filter(|(method, ..)| *method == Method::PUT)
+ .map(|(.., sent)| sent)
+ .collect(),
+ metrics: app.metrics.render(),
+ }
+}
+
+#[tokio::test]
+async fn relay_put_retries_a_buffered_body_after_a_dropped_connection() {
+ let body = Bytes::from_static(b"buffered upload bytes");
+
+ let relay =
+ relay_put_through_a_dropped_connection(body.clone(), Some(body.len() as u64), 4096).await;
+
+ assert_eq!(StatusCode::OK, relay.status);
+ assert_eq!(2, relay.accepted);
+ assert_eq!(vec![body], relay.stored_puts);
+ assert!(
+ relay
+ .metrics
+ .contains("fluxer_media_proxy_relay_upstream_retries_total 1"),
+ "{}",
+ relay.metrics
+ );
+}
+
+#[tokio::test]
+async fn relay_put_retries_a_spooled_body_after_a_dropped_connection() {
+ let body = Bytes::from_static(b"spooled upload bytes");
+
+ let relay = relay_put_through_a_dropped_connection(body.clone(), None, 4096).await;
+
+ assert_eq!(StatusCode::OK, relay.status);
+ assert_eq!(2, relay.accepted);
+ assert_eq!(vec![body], relay.stored_puts);
+ assert!(
+ relay
+ .metrics
+ .contains("fluxer_media_proxy_relay_upstream_retries_total 1"),
+ "{}",
+ relay.metrics
+ );
+}
+
+#[tokio::test]
+async fn relay_put_does_not_retry_a_body_too_large_to_buffer() {
+ let body = Bytes::from_static(b"streamed upload bytes");
+
+ let relay =
+ relay_put_through_a_dropped_connection(body.clone(), Some(body.len() as u64), 4).await;
+
+ assert_eq!(StatusCode::BAD_GATEWAY, relay.status);
+ assert_eq!(1, relay.accepted);
+ assert!(relay.stored_puts.is_empty());
+ assert!(
+ relay
+ .metrics
+ .contains("fluxer_media_proxy_relay_upstream_retries_total 0"),
+ "{}",
+ relay.metrics
+ );
+}
+
#[tokio::test]
async fn relay_put_omits_the_etag_when_the_store_returns_none() {
let tmp = tempfile::tempdir().unwrap();
diff --git a/fluxer_media_proxy/src/server/routes/relay.rs b/fluxer_media_proxy/src/server/routes/relay.rs
index 9c5394ddf..77306554b 100644
--- a/fluxer_media_proxy/src/server/routes/relay.rs
+++ b/fluxer_media_proxy/src/server/routes/relay.rs
@@ -2,18 +2,18 @@
use crate::{
config::DeploymentMode,
- external_path, http_headers,
+ external_path, http_client, http_headers,
server::{
relay::body::{
- RelayBodyProgress, RelayBodyStream, RelayBodyStreamRequest, relay_body_chunks,
- relay_etag, validate_completed_relay_body,
+ RelayBodyProgress, RelayBodyStream, RelayBodyStreamRequest, buffer_relay_body,
+ relay_body_chunks, relay_etag, validate_completed_relay_body,
},
response::error::text,
state::AppState,
},
spool::{SpoolError, spool_to_temp},
storage::{RelayBody, RelayPutOptions},
- upload_relay::{RelayError, target, token},
+ upload_relay::{BufferReservation, RelayError, target, token},
};
use axum::{
body::Body,
@@ -91,7 +91,7 @@ pub(in crate::server) async fn relay_put(
return relay_error(err);
}
let timeout_ms = relay_upstream_timeout_ms(&app, content_length);
- let (body, body_length, streamed) = match content_length {
+ let (body, body_length, streamed, _buffer_reservation) = match content_length {
Some(declared) => {
let Some(deadline) =
tokio::time::Instant::now().checked_add(Duration::from_millis(timeout_ms))
@@ -106,11 +106,34 @@ pub(in crate::server) async fn relay_put(
failure: Arc::clone(&failure),
});
let progress = stream.progress();
- (
- RelayBody::Streamed(relay_body_chunks(stream)),
- declared,
- Some(StreamedRelayBody { failure, progress }),
- )
+ match buffer_reservation(&app, declared) {
+ Some(reservation) => {
+ let Ok(buffered) = buffer_relay_body(relay_body_chunks(stream), declared).await
+ else {
+ return relay_error(
+ failure
+ .get()
+ .copied()
+ .unwrap_or(RelayError::ClientUploadFailed),
+ );
+ };
+ if let Err(err) = validate_completed_relay_body(&failure, &progress, declared) {
+ return relay_error(err);
+ }
+ (
+ RelayBody::Buffered(buffered),
+ declared,
+ None,
+ Some(reservation),
+ )
+ }
+ None => (
+ RelayBody::Streamed(relay_body_chunks(stream)),
+ declared,
+ Some(StreamedRelayBody { failure, progress }),
+ None,
+ ),
+ }
}
None => {
let body_length_limit = token.mb.min(app.cfg.upload_relay.max_body_bytes);
@@ -140,7 +163,7 @@ pub(in crate::server) async fn relay_put(
}
};
let (file, spooled_length) = spooled.into_parts();
- (RelayBody::Spooled(file), spooled_length, None)
+ (RelayBody::Spooled(file), spooled_length, None, None)
}
};
let content_type = resolve_relay_content_type(
@@ -149,19 +172,32 @@ pub(in crate::server) async fn relay_put(
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok()),
);
- let options = RelayPutOptions {
+ let upload_id = params.get("uploadId").cloned();
+ let relay_options = |body| RelayPutOptions {
body,
content_length: body_length,
- content_type,
- upload_id: params.get("uploadId").cloned(),
+ content_type: content_type.clone(),
+ upload_id: upload_id.clone(),
part_number,
timeout_ms,
};
- match app
+ let (body, replay) = with_replay(body).await;
+ let mut outcome = app
.store
- .relay_put_object(&app.cfg.storage.bucket_uploads, &key, options)
- .await
- {
+ .relay_put_object(&app.cfg.storage.bucket_uploads, &key, relay_options(body))
+ .await;
+ let retry_body = match &outcome {
+ Err(err) if http_client::is_dropped_connection(err) => replay,
+ _ => None,
+ };
+ if let Some(body) = retry_body {
+ app.metrics.relay().record_retry();
+ outcome = app
+ .store
+ .relay_put_object(&app.cfg.storage.bucket_uploads, &key, relay_options(body))
+ .await;
+ }
+ match outcome {
Ok(etag) => match relay_success_etag(etag, streamed.as_ref(), body_length) {
Ok(etag) => {
app.metrics.relay().record_success();
@@ -183,6 +219,30 @@ pub(in crate::server) async fn relay_put(
}
}
+fn buffer_reservation(app: &AppState, declared_length: u64) -> Option<BufferReservation> {
+ if declared_length > app.cfg.upload_relay.buffered_retry_max_bytes {
+ return None;
+ }
+ BufferReservation::try_new(
+ declared_length,
+ app.cfg.upload_relay.buffered_retry_total_bytes,
+ )
+}
+
+async fn with_replay(body: RelayBody) -> (RelayBody, Option<RelayBody>) {
+ match body {
+ RelayBody::Buffered(bytes) => (
+ RelayBody::Buffered(bytes.clone()),
+ Some(RelayBody::Buffered(bytes)),
+ ),
+ RelayBody::Spooled(file) => {
+ let replay = file.try_clone().await.ok().map(RelayBody::Spooled);
+ (RelayBody::Spooled(file), replay)
+ }
+ RelayBody::Streamed(chunks) => (RelayBody::Streamed(chunks), None),
+ }
+}
+
fn relay_upstream_timeout_ms(app: &AppState, content_length: Option<u64>) -> u64 {
let Some(declared) = content_length else {
return app.cfg.upload_relay.s3_timeout_ms;
diff --git a/fluxer_media_proxy/src/storage/local.rs b/fluxer_media_proxy/src/storage/local.rs
index 8f86b8a46..e08c8fb03 100644
--- a/fluxer_media_proxy/src/storage/local.rs
+++ b/fluxer_media_proxy/src/storage/local.rs
@@ -202,6 +202,9 @@ impl Store {
dest.write_all(&chunk?).await?;
}
}
+ RelayBody::Buffered(bytes) => {
+ dest.write_all(&bytes).await?;
+ }
}
dest.flush().await?;
dest.sync_all().await?;
diff --git a/fluxer_media_proxy/src/storage/relay_body.rs b/fluxer_media_proxy/src/storage/relay_body.rs
index fd34690d2..a4d35f518 100644
--- a/fluxer_media_proxy/src/storage/relay_body.rs
+++ b/fluxer_media_proxy/src/storage/relay_body.rs
@@ -17,6 +17,7 @@ pub type RelayBodyChunks =
pub enum RelayBody {
Spooled(tokio::fs::File),
Streamed(RelayBodyChunks),
+ Buffered(Bytes),
}
pub struct RelayPutOptions {
diff --git a/fluxer_media_proxy/src/storage/s3.rs b/fluxer_media_proxy/src/storage/s3.rs
index df4e8ac02..8a9f1b8eb 100644
--- a/fluxer_media_proxy/src/storage/s3.rs
+++ b/fluxer_media_proxy/src/storage/s3.rs
@@ -339,6 +339,7 @@ impl Store {
reqwest::Body::wrap(SizedFileBody::new(file, options.content_length))
}
RelayBody::Streamed(chunks) => reqwest::Body::wrap_stream(chunks),
+ RelayBody::Buffered(bytes) => reqwest::Body::from(bytes),
};
let response = self
.raw_client
diff --git a/fluxer_media_proxy/src/storage/tests/mod.rs b/fluxer_media_proxy/src/storage/tests/mod.rs
index b9b1e9aa2..82a085d31 100644
--- a/fluxer_media_proxy/src/storage/tests/mod.rs
+++ b/fluxer_media_proxy/src/storage/tests/mod.rs
@@ -21,7 +21,16 @@ use axum::response::Response;
use bytes::Bytes;
use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, header};
use parking_lot::Mutex;
-use std::{collections::HashMap, path::Path, sync::Arc, time::Duration};
+use std::{
+ collections::HashMap,
+ path::Path,
+ sync::{
+ Arc,
+ atomic::{AtomicUsize, Ordering},
+ },
+ time::Duration,
+};
+use tokio::io::AsyncReadExt as _;
pub(crate) type CapturedRequest = (Method, http::Uri, HeaderMap, Bytes);
@@ -287,3 +296,30 @@ fn fake_s3_response(
}
response
}
+
+pub(crate) async fn connection_dropping_front(
+ origin: &str,
+ dropped: usize,
+) -> (String, Arc<AtomicUsize>) {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let endpoint = format!("http://{}", listener.local_addr().unwrap());
+ let origin = origin.trim_start_matches("http://").to_owned();
+ let accepted = Arc::new(AtomicUsize::new(0));
+ let counter = Arc::clone(&accepted);
+ tokio::spawn(async move {
+ while let Ok((mut client, _)) = listener.accept().await {
+ let index = counter.fetch_add(1, Ordering::SeqCst);
+ let origin = origin.clone();
+ tokio::spawn(async move {
+ if index < dropped {
+ let mut request = [0u8; 4096];
+ let _ = client.read(&mut request).await;
+ return;
+ }
+ let mut upstream = tokio::net::TcpStream::connect(origin).await.unwrap();
+ let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await;
+ });
+ }
+ });
+ (endpoint, accepted)
+}
diff --git a/fluxer_media_proxy/src/storage/tests/s3.rs b/fluxer_media_proxy/src/storage/tests/s3.rs
index 14372a003..86d122a6a 100644
--- a/fluxer_media_proxy/src/storage/tests/s3.rs
+++ b/fluxer_media_proxy/src/storage/tests/s3.rs
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-use super::{FakeObject, FakeS3, fake_s3, store};
+use super::{FakeObject, FakeS3, connection_dropping_front, fake_s3, store};
use crate::{
byte_budget::ByteBudget,
range::ByteRange,
@@ -8,14 +8,7 @@ use crate::{
storage::{ObjectReadRequest, ObjectStreamRequest, StorageError},
};
use http::{Method, StatusCode, header};
-use std::{
- sync::{
- Arc,
- atomic::{AtomicUsize, Ordering},
- },
- time::Duration,
-};
-use tokio::io::AsyncReadExt as _;
+use std::{sync::atomic::Ordering, time::Duration};
const LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT";
@@ -393,30 +386,6 @@ async fn a_read_that_keeps_losing_its_connection_reports_the_cause() {
assert!(fake.requests().is_empty());
}
-async fn connection_dropping_front(origin: &str, dropped: usize) -> (String, Arc<AtomicUsize>) {
- let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
- let endpoint = format!("http://{}", listener.local_addr().unwrap());
- let origin = origin.trim_start_matches("http://").to_owned();
- let accepted = Arc::new(AtomicUsize::new(0));
- let counter = Arc::clone(&accepted);
- tokio::spawn(async move {
- while let Ok((mut client, _)) = listener.accept().await {
- let index = counter.fetch_add(1, Ordering::SeqCst);
- let origin = origin.clone();
- tokio::spawn(async move {
- if index < dropped {
- let mut request = [0u8; 4096];
- let _ = client.read(&mut request).await;
- return;
- }
- let mut upstream = tokio::net::TcpStream::connect(origin).await.unwrap();
- let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await;
- });
- }
- });
- (endpoint, accepted)
-}
-
fn fake_gets(fake: &FakeS3, path: &str) -> usize {
fake.requests()
.iter()
diff --git a/fluxer_media_proxy/src/upload_relay/mod.rs b/fluxer_media_proxy/src/upload_relay/mod.rs
index a4ba2ad79..a31ac1b63 100644
--- a/fluxer_media_proxy/src/upload_relay/mod.rs
+++ b/fluxer_media_proxy/src/upload_relay/mod.rs
@@ -53,6 +53,22 @@ pub fn release_buffer_budget(amount: u64) {
BUFFERED_RETRY_IN_FLIGHT.fetch_sub(amount, Ordering::AcqRel);
}
+pub struct BufferReservation {
+ bytes: u64,
+}
+
+impl BufferReservation {
+ pub fn try_new(bytes: u64, ceiling: u64) -> Option<Self> {
+ try_reserve_buffer_budget(bytes, ceiling).then(|| Self { bytes })
+ }
+}
+
+impl Drop for BufferReservation {
+ fn drop(&mut self) {
+ release_buffer_budget(self.bytes);
+ }
+}
+
pub fn try_reserve_spool_budget(needed: u64, ceiling: u64) -> bool {
try_reserve(&SPOOL_IN_FLIGHT_BYTES, needed, ceiling)
}
diff --git a/fluxer_media_proxy/src/upload_relay/tests.rs b/fluxer_media_proxy/src/upload_relay/tests.rs
index c037bb3e3..d0bf6f0b6 100644
--- a/fluxer_media_proxy/src/upload_relay/tests.rs
+++ b/fluxer_media_proxy/src/upload_relay/tests.rs
@@ -1,12 +1,13 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::upload_relay::{
- RelayError, release_buffer_budget,
+ RelayError,
target::{RelayRequest, query_part_number, valid_content_type, validate_relay_request},
token::{TokenError, TokenMethod, TokenPayload, decode_token, encode_token, token_from_query},
- try_reserve_buffer_budget,
+ try_reserve,
};
use http::Method;
+use std::sync::atomic::{AtomicU64, Ordering};
const BASE64_URL_ALPHABET: &[u8] =
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
@@ -124,11 +125,12 @@ fn validates_relay_mismatches() {
#[test]
fn buffer_budget_is_bounded() {
- assert!(try_reserve_buffer_budget(4, 8));
- assert!(!try_reserve_buffer_budget(5, 8));
- release_buffer_budget(4);
- assert!(try_reserve_buffer_budget(8, 8));
- release_buffer_budget(8);
+ let in_flight = AtomicU64::new(0);
+ assert!(try_reserve(&in_flight, 4, 8));
+ assert!(!try_reserve(&in_flight, 5, 8));
+ in_flight.fetch_sub(4, Ordering::AcqRel);
+ assert!(try_reserve(&in_flight, 8, 8));
+ assert!(!try_reserve(&AtomicU64::new(0), 1, 0));
}
#[test]