A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::common::{env_bool, env_string};
use anyhow::{Context, Result, bail, ensure};
use aws_config::{BehaviorVersion, Region, retry::RetryConfig};
use aws_sdk_s3::Client as S3Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64;
use md5::{Digest as _, Md5};
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use std::fs::{self};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Semaphore;
use tokio::task::JoinSet;
const DEFAULT_S3_WRITE_CONCURRENCY: usize = 8;
const DEFAULT_S3_RETRY_ATTEMPTS: u32 = 8;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct S3UploadPlanItem {
pub(crate) path: PathBuf,
pub(crate) key: String,
pub(crate) content_type: Option<String>,
pub(crate) cache_control: Option<String>,
pub(crate) repair_existing_metadata: bool,
}
impl S3UploadPlanItem {
pub(crate) fn new(path: PathBuf, key: String) -> Self {
Self {
path,
key,
content_type: None,
cache_control: None,
repair_existing_metadata: false,
}
}
pub(crate) fn with_content_type(mut self, content_type: impl Into<String>) -> Self {
self.content_type = Some(content_type.into());
self
}
pub(crate) fn with_cache_control(mut self, cache_control: impl Into<String>) -> Self {
self.cache_control = Some(cache_control.into());
self
}
pub(crate) fn with_detected_content_type(self) -> Self {
let Some(content_type) = s3_content_type_for_key(&self.key) else {
return self;
};
self.with_content_type(content_type)
}
pub(crate) fn repair_existing_metadata(mut self) -> Self {
self.repair_existing_metadata = true;
self
}
fn has_upload_metadata(&self) -> bool {
self.content_type.is_some() || self.cache_control.is_some()
}
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub(crate) struct S3UploadStats {
pub(crate) uploaded: usize,
pub(crate) skipped_existing: usize,
pub(crate) metadata_repaired: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum S3UploadDisposition {
Uploaded,
SkippedExisting,
MetadataRepaired,
}
pub(crate) async fn s3_client(default_endpoint: Option<&str>) -> Result<S3Client> {
let region = env::var("AWS_REGION")
.or_else(|_| env::var("AWS_DEFAULT_REGION"))
.unwrap_or_else(|_| "us-east-1".to_string());
let mut loader = aws_config::defaults(BehaviorVersion::latest()).region(Region::new(region));
let endpoint = env_string("S3_ENDPOINT").or_else(|| default_endpoint.map(ToOwned::to_owned));
if let Some(endpoint) = endpoint.as_deref() {
loader = loader.endpoint_url(endpoint);
}
loader = loader.retry_config(s3_retry_config());
let shared_config = loader.load().await;
let mut s3_config = aws_sdk_s3::config::Builder::from(&shared_config);
if endpoint.is_some() || env_bool("S3_FORCE_PATH_STYLE") {
s3_config = s3_config.force_path_style(true);
}
Ok(S3Client::from_conf(s3_config.build()))
}
pub(crate) async fn upload_s3_plan_append_only(
client: &S3Client,
bucket: &str,
plan: Vec<S3UploadPlanItem>,
) -> Result<S3UploadStats> {
ensure_unique_s3_keys(&plan)?;
let existing_objects = existing_s3_objects_for_plan(client, bucket, &plan).await?;
let mut stats = S3UploadStats::default();
let (pending, skipped_items) = split_append_only_upload_plan(bucket, plan, &existing_objects)?;
let concurrency = s3_write_concurrency();
let semaphore = Arc::new(Semaphore::new(concurrency));
let bucket = bucket.to_string();
let mut tasks = JoinSet::new();
for item in pending {
let permit = semaphore
.clone()
.acquire_owned()
.await
.context("S3 upload semaphore closed")?;
let client = client.clone();
let bucket = bucket.clone();
tasks.spawn(async move {
let _permit = permit;
let disposition = put_file_to_s3(&client, &bucket, &item, true)
.await
.with_context(|| {
format!("Failed append-only upload for s3://{}/{}", bucket, item.key)
})?;
Ok::<_, anyhow::Error>(disposition)
});
}
for item in skipped_items {
if item.repair_existing_metadata && item.has_upload_metadata() {
let permit = semaphore
.clone()
.acquire_owned()
.await
.context("S3 upload semaphore closed")?;
let client = client.clone();
let bucket = bucket.clone();
tasks.spawn(async move {
let _permit = permit;
put_file_to_s3(&client, &bucket, &item, false)
.await
.with_context(|| {
format!("Failed to repair metadata for s3://{}/{}", bucket, item.key)
})
});
} else {
println!("Skipping existing s3://{bucket}/{}", item.key);
stats.skipped_existing += 1;
}
}
while let Some(result) = tasks.join_next().await {
match result.context("S3 upload task failed")?? {
S3UploadDisposition::Uploaded => stats.uploaded += 1,
S3UploadDisposition::SkippedExisting => stats.skipped_existing += 1,
S3UploadDisposition::MetadataRepaired => stats.metadata_repaired += 1,
}
}
Ok(stats)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct S3ObjectMetadata {
pub(crate) e_tag: Option<String>,
pub(crate) size: Option<i64>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct S3ListedObject {
key: String,
metadata: S3ObjectMetadata,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct S3FileIdentity {
size: u64,
md5_hex: String,
md5_base64: String,
}
async fn existing_s3_objects_for_plan(
client: &S3Client,
bucket: &str,
plan: &[S3UploadPlanItem],
) -> Result<BTreeMap<String, S3ObjectMetadata>> {
let planned_keys = plan
.iter()
.map(|item| item.key.clone())
.collect::<BTreeSet<_>>();
let prefixes = s3_list_prefixes_for_plan(plan);
let mut existing = BTreeMap::new();
for prefix in prefixes {
println!("Scanning existing S3 objects under s3://{bucket}/{prefix}");
for object in list_s3_objects(client, bucket, &prefix).await? {
if planned_keys.contains(&object.key) {
existing.insert(object.key, object.metadata);
}
}
}
Ok(existing)
}
pub(crate) fn split_append_only_upload_plan(
bucket: &str,
plan: Vec<S3UploadPlanItem>,
existing_objects: &BTreeMap<String, S3ObjectMetadata>,
) -> Result<(Vec<S3UploadPlanItem>, Vec<S3UploadPlanItem>)> {
ensure_unique_s3_keys(&plan)?;
let mut pending = Vec::new();
let mut skipped = Vec::new();
for item in plan {
if let Some(remote) = existing_objects.get(&item.key) {
if let Err(error) = ensure_existing_s3_object_matches_local(bucket, &item, remote) {
if !item.key.ends_with(".map") {
return Err(error);
}
println!("Keeping the published source map: {error}");
}
skipped.push(item);
} else {
pending.push(item);
}
}
Ok((pending, skipped))
}
fn ensure_existing_s3_object_matches_local(
bucket: &str,
item: &S3UploadPlanItem,
remote: &S3ObjectMetadata,
) -> Result<()> {
let local = s3_file_identity(&item.path)?;
if let Some(remote_size) = remote.size {
ensure!(
remote_size >= 0,
"Existing S3 object has invalid negative size: s3://{bucket}/{}",
item.key
);
ensure!(
remote_size as u64 == local.size,
"Existing S3 object differs from local file: s3://{bucket}/{} has {} byte(s), local {} has {} byte(s)",
item.key,
remote_size,
item.path.display(),
local.size
);
}
let Some(e_tag) = remote.e_tag.as_deref() else {
bail!(
"Existing S3 object cannot be verified because ListObjectsV2 did not return an ETag: s3://{bucket}/{}",
item.key
);
};
let Some(remote_md5) = s3_etag_md5(e_tag) else {
bail!(
"Existing S3 object cannot be verified because its ETag is not a single-part MD5: s3://{bucket}/{} (ETag {e_tag})",
item.key
);
};
ensure!(
remote_md5 == local.md5_hex,
"Existing S3 object differs from local file: s3://{bucket}/{} has ETag {e_tag}, local {} has MD5 {}",
item.key,
item.path.display(),
local.md5_hex
);
Ok(())
}
fn s3_file_identity(path: &Path) -> Result<S3FileIdentity> {
let mut file =
fs::File::open(path).with_context(|| format!("Failed to read {}", path.display()))?;
let size = file
.metadata()
.with_context(|| format!("Failed to stat {}", path.display()))?
.len();
let mut hasher = Md5::new();
let mut buffer = [0_u8; 64 * 1024];
loop {
let read = file
.read(&mut buffer)
.with_context(|| format!("Failed to read {}", path.display()))?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
}
let digest = hasher.finalize();
Ok(S3FileIdentity {
size,
md5_hex: hex::encode(digest),
md5_base64: BASE64.encode(digest),
})
}
fn s3_etag_md5(e_tag: &str) -> Option<String> {
let value = e_tag.trim().trim_matches('"');
if value.len() == 32 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) {
Some(value.to_ascii_lowercase())
} else {
None
}
}
fn s3_list_prefixes_for_plan(plan: &[S3UploadPlanItem]) -> BTreeSet<String> {
plan.iter()
.map(|item| s3_list_prefix_for_key(&item.key))
.collect()
}
fn s3_list_prefix_for_key(key: &str) -> String {
key.find('/')
.map(|index| key[..=index].to_string())
.unwrap_or_else(|| key.to_string())
}
fn ensure_unique_s3_keys(plan: &[S3UploadPlanItem]) -> Result<()> {
let mut keys = BTreeSet::new();
for item in plan {
ensure!(
keys.insert(item.key.clone()),
"Duplicate S3 upload key in local plan: {}",
item.key
);
}
Ok(())
}
async fn put_file_to_s3(
client: &S3Client,
bucket: &str,
item: &S3UploadPlanItem,
if_absent: bool,
) -> Result<S3UploadDisposition> {
let (action, operation, failure, done) = if if_absent {
(
"Uploading",
"upload",
"upload",
S3UploadDisposition::Uploaded,
)
} else {
(
"Repairing metadata for",
"metadata repair",
"repair metadata for",
S3UploadDisposition::MetadataRepaired,
)
};
println!(
"{action} {} -> s3://{bucket}/{}",
item.path.display(),
item.key
);
let identity = s3_file_identity(&item.path)?;
let attempts = s3_retry_attempts();
for attempt in 1..=attempts {
let body = ByteStream::from_path(&item.path)
.await
.with_context(|| format!("Failed to read {}", item.path.display()))?;
let mut request = client
.put_object()
.bucket(bucket)
.key(&item.key)
.set_if_none_match(if_absent.then(|| "*".to_string()))
.content_md5(identity.md5_base64.clone())
.body(body);
if let Some(content_type) = &item.content_type {
request = request.content_type(content_type);
}
if let Some(cache_control) = &item.cache_control {
request = request.cache_control(cache_control);
}
match request.send().await {
Ok(_) => return Ok(done),
Err(error) => {
let code = error
.as_service_error()
.and_then(|error| error.code())
.map(ToOwned::to_owned);
let status = error
.raw_response()
.map(|response| response.status().as_u16());
if if_absent && is_existing_object_error(code.as_deref(), status) {
println!("Skipping existing s3://{bucket}/{}", item.key);
return Ok(S3UploadDisposition::SkippedExisting);
}
if is_retryable_s3_error(code.as_deref(), status) && attempt < attempts {
sleep_before_s3_retry(
operation,
&format!("s3://{bucket}/{}", item.key),
attempt,
attempts,
code.as_deref(),
status,
)
.await;
continue;
}
let summary = s3_error_summary(code.as_deref(), status);
return Err(error).with_context(|| {
format!("Failed to {failure} s3://{bucket}/{}{summary}", item.key)
});
}
}
}
unreachable!("S3 retry attempts are always greater than zero")
}
fn is_existing_object_error(code: Option<&str>, status: Option<u16>) -> bool {
matches!(
code,
Some("PreconditionFailed" | "ConditionalRequestConflict")
) || matches!(status, Some(409 | 412))
}
fn is_retryable_s3_error(code: Option<&str>, status: Option<u16>) -> bool {
status == Some(429) || matches!(code, Some("RateLimitExceeded" | "TooManyRequests"))
}
async fn sleep_before_s3_retry(
operation: &str,
target: &str,
attempt: u32,
attempts: u32,
code: Option<&str>,
status: Option<u16>,
) {
let delay = s3_retry_delay(attempt);
let summary = s3_error_summary(code, status);
println!(
"Retrying S3 {operation} {target}{summary} in {:.1}s (attempt {}/{attempts})",
delay.as_secs_f32(),
attempt + 1
);
tokio::time::sleep(delay).await;
}
fn s3_retry_delay(attempt: u32) -> Duration {
let shift = attempt.saturating_sub(1).min(5);
Duration::from_millis(500 * (1_u64 << shift))
}
fn s3_error_summary(code: Option<&str>, status: Option<u16>) -> String {
match (code, status) {
(Some(code), Some(status)) => format!(" (S3 code {code}, HTTP status {status})"),
(Some(code), None) => format!(" (S3 code {code})"),
(None, Some(status)) => format!(" (HTTP status {status})"),
(None, None) => String::new(),
}
}
fn s3_retry_config() -> RetryConfig {
RetryConfig::standard()
.with_max_attempts(s3_retry_attempts())
.with_initial_backoff(Duration::from_millis(500))
.with_max_backoff(Duration::from_secs(30))
}
fn s3_retry_attempts() -> u32 {
env::var("S3_RETRY_ATTEMPTS")
.ok()
.and_then(|value| value.parse::<u32>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_S3_RETRY_ATTEMPTS)
}
fn s3_write_concurrency() -> usize {
env::var("S3_WRITE_CONCURRENCY")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_S3_WRITE_CONCURRENCY)
}
pub(crate) fn s3_content_type_for_key(key: &str) -> Option<&'static str> {
let ext = key.rsplit('.').next()?.to_ascii_lowercase();
match ext.as_str() {
"html" | "htm" => Some("text/html; charset=utf-8"),
"js" | "mjs" => Some("application/javascript; charset=utf-8"),
"css" => Some("text/css; charset=utf-8"),
"json" | "map" => Some("application/json; charset=utf-8"),
"wasm" => Some("application/wasm"),
"png" => Some("image/png"),
"jpg" | "jpeg" => Some("image/jpeg"),
"gif" => Some("image/gif"),
"webp" => Some("image/webp"),
"avif" => Some("image/avif"),
"svg" => Some("image/svg+xml"),
"ico" => Some("image/x-icon"),
"woff" => Some("font/woff"),
"woff2" => Some("font/woff2"),
"ttf" => Some("font/ttf"),
"otf" => Some("font/otf"),
"eot" => Some("application/vnd.ms-fontobject"),
"mp3" => Some("audio/mpeg"),
"mp4" => Some("video/mp4"),
"webm" => Some("video/webm"),
"ogg" => Some("audio/ogg"),
"wav" => Some("audio/wav"),
"pdf" => Some("application/pdf"),
"txt" => Some("text/plain; charset=utf-8"),
"xml" => Some("application/xml; charset=utf-8"),
"webmanifest" => Some("application/manifest+json"),
_ => None,
}
}
async fn list_s3_objects(
client: &S3Client,
bucket: &str,
prefix: &str,
) -> Result<Vec<S3ListedObject>> {
let mut objects = Vec::new();
let mut token: Option<String> = None;
loop {
let response =
send_s3_list_objects_v2_page(client, bucket, prefix, token.as_deref()).await?;
objects.extend(response.contents().iter().filter_map(|object| {
object.key().map(|key| S3ListedObject {
key: key.to_string(),
metadata: S3ObjectMetadata {
e_tag: object.e_tag().map(ToOwned::to_owned),
size: object.size(),
},
})
}));
token = response.next_continuation_token().map(ToOwned::to_owned);
if token.is_none() {
break;
}
}
Ok(objects)
}
async fn send_s3_list_objects_v2_page(
client: &S3Client,
bucket: &str,
prefix: &str,
token: Option<&str>,
) -> Result<aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output> {
let attempts = s3_retry_attempts();
for attempt in 1..=attempts {
let mut request = client.list_objects_v2().bucket(bucket).prefix(prefix);
if let Some(value) = token {
request = request.continuation_token(value);
}
match request.send().await {
Ok(response) => return Ok(response),
Err(error) => {
let code = error
.as_service_error()
.and_then(|error| error.code())
.map(ToOwned::to_owned);
let status = error
.raw_response()
.map(|response| response.status().as_u16());
if is_retryable_s3_error(code.as_deref(), status) && attempt < attempts {
sleep_before_s3_retry(
"list",
&format!("s3://{bucket}/{prefix}"),
attempt,
attempts,
code.as_deref(),
status,
)
.await;
continue;
}
let summary = s3_error_summary(code.as_deref(), status);
return Err(error)
.with_context(|| format!("Failed to list s3://{bucket}/{prefix}{summary}"));
}
}
}
unreachable!("S3 retry attempts are always greater than zero")
}