Skip to content
Python
7.4 KB
Raw Blame
use crate::*;
use object_store::{
    aws::{AmazonS3, AmazonS3Builder, AwsCredentialProvider, S3ConditionalPut},
    ClientOptions, ObjectStore, ObjectStoreExt, PutMode, PutOptions, RetryConfig,
};
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Config {
    pub deployment: String,
    pub bucket: String,
    pub region: String,
    pub endpoint: Option<String>,
    /// Explicit trust anchor for a private HTTPS object-store endpoint.
    #[serde(default)]
    pub ca_file: Option<std::path::PathBuf>,
}
pub struct S3 {
    client: AmazonS3,
}
fn map(e: object_store::Error) -> anyhow::Error {
    match e {
        object_store::Error::NotFound { .. } => Error::NotFound.into(),
        object_store::Error::AlreadyExists { .. } | object_store::Error::Precondition { .. } => {
            Error::Conflict.into()
        }
        _ => Error::Unavailable.into(),
    }
}
impl Config {
    pub fn build(&self, credentials: Option<AwsCredentialProvider>) -> Result<Store> {
        // reqwest's no-provider feature deliberately requires an explicit choice.
        // Preserve an embedding application's already-installed crypto provider.
        if rustls::crypto::CryptoProvider::get_default().is_none() {
            let _ = rustls::crypto::ring::default_provider().install_default();
        }
        component(&self.deployment, 64)?;
        component(&self.region, 64)?;
        ensure!(
            (3..=63).contains(&self.bucket.len())
                && self
                    .bucket
                    .bytes()
                    .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'-'),
            Error::Invalid
        );
        let mut builder = AmazonS3Builder::new()
            .with_bucket_name(&self.bucket)
            .with_region(&self.region)
            .with_virtual_hosted_style_request(false)
            .with_conditional_put(S3ConditionalPut::ETagMatch)
            .with_retry(RetryConfig {
                max_retries: 0,
                retry_timeout: Duration::from_secs(15),
                ..Default::default()
            });
        let mut allow_http = false;
        let endpoint = if let Some(endpoint) = &self.endpoint {
            let url = url::Url::parse(endpoint).map_err(|_| Error::Invalid)?;
            ensure!(
                url.username().is_empty()
                    && url.password().is_none()
                    && url.query().is_none()
                    && url.fragment().is_none()
                    && url.path() == "/",
                Error::Invalid
            );
            let local = url.host_str().is_some_and(|h| {
                h == "localhost"
                    || h.trim_matches(['[', ']'])
                        .parse::<std::net::IpAddr>()
                        .is_ok_and(|ip| ip.is_loopback())
            });
            allow_http = url.scheme() == "http" && local;
            ensure!(url.scheme() == "https" || allow_http, Error::Invalid);
            let endpoint = url.to_string();
            builder = builder.with_endpoint(&endpoint);
            endpoint
        } else {
            format!("https://s3.{}.amazonaws.com/", self.region)
        };
        let mut options = ClientOptions::new()
            .with_allow_http(allow_http)
            .with_allow_invalid_certificates(false)
            .with_timeout(Duration::from_secs(15))
            .with_connect_timeout(Duration::from_secs(3));
        if let Some(path) = &self.ca_file {
            use std::io::Read;
            let mut pem = Vec::new();
            std::fs::File::open(path)
                .map_err(|_| Error::Invalid)?
                .take(65_537)
                .read_to_end(&mut pem)
                .map_err(|_| Error::Invalid)?;
            ensure!(pem.len() <= 65_536, Error::Invalid);
            let certificates =
                object_store::Certificate::from_pem_bundle(&pem).map_err(|_| Error::Invalid)?;
            ensure!(!certificates.is_empty(), Error::Invalid);
            for certificate in certificates {
                options = options.with_root_certificate(certificate);
            }
        }
        builder = builder.with_client_options(options);
        if let Some(credentials) = credentials {
            builder = builder.with_credentials(credentials);
        } else {
            // An explicitly configured Pod Identity source must be complete.
            // A missing token-file/endpoint must never fall through to the node's
            // instance role, which may have a different permission boundary.
            ensure!(
                std::env::var_os("AWS_CONTAINER_CREDENTIALS_FULL_URI").is_some()
                    == std::env::var_os("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE").is_some(),
                Error::Invalid
            );
            // Deliberately import credential inputs only, never ambient bucket,
            // endpoint, HTTP/TLS policy or arbitrary AWS configuration overrides.
            for key in [
                "AWS_ACCESS_KEY_ID",
                "AWS_SECRET_ACCESS_KEY",
                "AWS_SESSION_TOKEN",
                "AWS_WEB_IDENTITY_TOKEN_FILE",
                "AWS_ROLE_ARN",
                "AWS_ROLE_SESSION_NAME",
                "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
                "AWS_CONTAINER_CREDENTIALS_FULL_URI",
                "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
            ] {
                if let Ok(value) = std::env::var(key) {
                    builder = builder.with_config(
                        key.to_ascii_lowercase()
                            .parse()
                            .map_err(|_| Error::Invalid)?,
                        value,
                    );
                }
            }
        }
        let identity = hash(
            format!(
                "s3-chunks-v1:{}:{}:{}:{}",
                self.deployment, self.bucket, self.region, endpoint
            )
            .as_bytes(),
        );
        Store::new(
            Arc::new(S3 {
                client: builder.build().map_err(|_| Error::Unavailable)?,
            }),
            Binding {
                deployment: self.deployment.clone(),
                identity,
            },
        )
    }
}
#[async_trait::async_trait]
impl Backend for S3 {
    async fn create(&self, key: &str, bytes: Bytes) -> Result<()> {
        ensure!(bytes.len() <= CHUNK, Error::TooLarge);
        let path = object_store::path::Path::parse(key).map_err(|_| Error::Invalid)?;
        self.client
            .put_opts(
                &path,
                bytes.into(),
                PutOptions {
                    mode: PutMode::Create,
                    ..Default::default()
                },
            )
            .await
            .map_err(map)?;
        Ok(())
    }
    async fn read(&self, key: &str, limit: usize) -> Result<Bytes> {
        ensure!(limit <= CHUNK, Error::TooLarge);
        let path = object_store::path::Path::parse(key).map_err(|_| Error::Invalid)?;
        let result = self.client.get(&path).await.map_err(map)?;
        ensure!(result.meta.size <= limit as u64, Error::Conflict);
        let mut input = result.into_stream();
        let mut bytes = Vec::with_capacity(limit.min(CHUNK));
        while let Some(chunk) = input.next().await {
            let chunk = chunk.map_err(map)?;
            ensure!(bytes.len() + chunk.len() <= limit, Error::Conflict);
            bytes.extend_from_slice(&chunk);
        }
        Ok(bytes.into())
    }
}