Skip to content
Python
22.4 KB
Raw Blame
//! Azure Blob artifacts with an explicit AKS workload identity. Configuration is
//! host-owned; no shared keys, SAS, ambient Azure settings or IMDS fallback.
use crate::*;
use object_store::{
    azure::{AzureCredentialProvider, MicrosoftAzure, MicrosoftAzureBuilder},
    client::{HttpClient, HttpConnector},
    ClientOptions, ObjectStore, ObjectStoreExt, PutMode, PutOptions, RetryConfig,
};
use std::path::PathBuf;

#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Config {
    pub deployment: String,
    pub account: String,
    pub container: String,
    pub tenant_id: String,
    pub client_id: String,
    pub federated_token_file: PathBuf,
}

pub struct Azure {
    client: MicrosoftAzure,
}

// object_store's default HTTP client follows redirects. In particular a 307 PUT
// must never forward a write or a token to a different endpoint. Disable both
// redirect following and transport-level retries, including the OAuth client.
#[derive(Debug)]
struct Connector(reqwest013::Client);
impl HttpConnector for Connector {
    fn connect(&self, _: &ClientOptions) -> object_store::Result<HttpClient> {
        Ok(HttpClient::new(self.0.clone()))
    }
}

fn map(error: object_store::Error) -> anyhow::Error {
    match error {
        object_store::Error::NotFound { .. } => Error::NotFound.into(),
        object_store::Error::AlreadyExists { .. } | object_store::Error::Precondition { .. } => {
            Error::Conflict.into()
        }
        _ => Error::Unavailable.into(),
    }
}

impl Config {
    fn validate(&self) -> Result<()> {
        component(&self.deployment, 64)?;
        ensure!(
            (3..=24).contains(&self.account.len())
                && self
                    .account
                    .bytes()
                    .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()),
            Error::Invalid
        );
        ensure!(
            (3..=63).contains(&self.container.len())
                && self
                    .container
                    .bytes()
                    .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'-')
                && !self.container.starts_with('-')
                && !self.container.ends_with('-')
                && !self.container.contains("--"),
            Error::Invalid
        );
        for id in [&self.tenant_id, &self.client_id] {
            let uuid = uuid::Uuid::parse_str(id).map_err(|_| Error::Invalid)?;
            ensure!(
                uuid.to_string() == id.to_ascii_lowercase() && !uuid.is_nil(),
                Error::Invalid
            );
        }
        // Do not open it here: controllers share configuration but receive no
        // token. Projected token files may be symlinks and rotate in-place.
        ensure!(
            self.federated_token_file.is_absolute()
                && self.federated_token_file.to_str().is_some()
                && !self
                    .federated_token_file
                    .components()
                    .any(|c| matches!(c, std::path::Component::ParentDir)),
            Error::Invalid
        );
        Ok(())
    }

    pub fn build(&self) -> Result<Store> {
        self.build_inner(
            &format!("https://{}.blob.core.windows.net", self.account),
            None,
            false,
            None,
        )
    }

    fn build_inner(
        &self,
        endpoint: &str,
        credentials: Option<AzureCredentialProvider>,
        fixture_http: bool,
        fixture_authority: Option<&str>,
    ) -> Result<Store> {
        self.validate()?;
        if rustls::crypto::CryptoProvider::get_default().is_none() {
            let _ = rustls::crypto::ring::default_provider().install_default();
        }
        let client = reqwest013::Client::builder()
            .redirect(reqwest013::redirect::Policy::none())
            .retry(reqwest013::retry::never())
            .no_proxy()
            .https_only(!fixture_http)
            .timeout(Duration::from_secs(15))
            .connect_timeout(Duration::from_secs(3))
            .no_gzip()
            .no_brotli()
            .no_zstd()
            .no_deflate()
            .build()
            .map_err(|_| Error::Unavailable)?;
        let mut builder = MicrosoftAzureBuilder::new()
            .with_account(&self.account)
            .with_container_name(&self.container)
            .with_endpoint(endpoint.to_owned())
            .with_tenant_id(&self.tenant_id)
            .with_client_id(&self.client_id)
            .with_federated_token_file(self.federated_token_file.to_str().ok_or(Error::Invalid)?)
            .with_credential_type("workload_identity")
            .with_authority_host(fixture_authority.unwrap_or("https://login.microsoftonline.com"))
            .with_http_connector(Connector(client))
            .with_client_options(ClientOptions::new().with_allow_http(fixture_http))
            // A failed write may have reached storage. Only an explicit retry
            // reconciles identical immutable bytes through the Store contract.
            .with_retry(RetryConfig {
                max_retries: 0,
                retry_timeout: Duration::from_secs(15),
                ..Default::default()
            });
        if let Some(credentials) = credentials {
            builder = builder.with_credentials(credentials);
        }
        let identity = hash(
            format!(
                "azure-chunks-v1:{}:{}:{}:{}:{}:{}",
                self.deployment,
                self.account,
                self.container,
                endpoint,
                self.tenant_id.to_ascii_lowercase(),
                self.client_id.to_ascii_lowercase()
            )
            .as_bytes(),
        );
        Store::new(
            Arc::new(Azure {
                client: builder.build().map_err(|_| Error::Unavailable)?,
            }),
            Binding {
                deployment: self.deployment.clone(),
                identity,
            },
        )
    }
}

#[async_trait::async_trait]
impl Backend for Azure {
    async fn create(&self, key: &str, bytes: Bytes) -> Result<()> {
        ensure!(bytes.len() <= CHUNK, Error::TooLarge);
        let path = object_store::path::Path::parse(key).map_err(|_| Error::Invalid)?;
        self.client
            .put_opts(
                &path,
                bytes.into(),
                PutOptions {
                    mode: PutMode::Create,
                    ..Default::default()
                },
            )
            .await
            .map_err(map)?;
        Ok(())
    }
    async fn read(&self, key: &str, limit: usize) -> Result<Bytes> {
        ensure!(limit <= CHUNK, Error::TooLarge);
        let path = object_store::path::Path::parse(key).map_err(|_| Error::Invalid)?;
        let result = self.client.get(&path).await.map_err(map)?;
        ensure!(result.meta.size <= limit as u64, Error::Conflict);
        let mut input = result.into_stream();
        let mut bytes = Vec::with_capacity(limit.min(CHUNK));
        while let Some(chunk) = input.next().await {
            let chunk = chunk.map_err(map)?;
            ensure!(bytes.len() + chunk.len() <= limit, Error::Conflict);
            bytes.extend_from_slice(&chunk);
        }
        Ok(bytes.into())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use axum::{
        body::to_bytes,
        extract::{Request, State},
        http::{header, StatusCode},
        response::{IntoResponse, Response},
        routing::any,
        Json, Router,
    };
    use futures_util::TryStreamExt;
    use object_store::{azure::AzureCredential, StaticCredentialProvider};
    use std::{
        collections::HashMap,
        sync::{
            atomic::{AtomicUsize, Ordering},
            Mutex,
        },
    };

    #[derive(Clone, Default)]
    struct Fixture {
        objects: Arc<Mutex<HashMap<String, Bytes>>>,
        calls: Arc<AtomicUsize>,
        mode: Arc<AtomicUsize>,
        tokens: Arc<AtomicUsize>,
        projected: Arc<Mutex<String>>,
    }
    async fn handle(State(f): State<Fixture>, request: Request) -> Response {
        if request.uri().path().ends_with("/oauth2/v2.0/token") {
            f.tokens.fetch_add(1, Ordering::SeqCst);
            if f.mode.load(Ordering::SeqCst) == 6 {
                return (
                    StatusCode::TEMPORARY_REDIRECT,
                    [(header::LOCATION, "/token-redirected")],
                )
                    .into_response();
            }
            if f.mode.load(Ordering::SeqCst) == 7 {
                return StatusCode::SERVICE_UNAVAILABLE.into_response();
            }
            let body = to_bytes(request.into_body(), 8192).await.unwrap();
            let body = String::from_utf8(body.to_vec()).unwrap();
            assert!(body.contains("grant_type=client_credentials"));
            assert!(body.contains("scope=https%3A%2F%2Fstorage.azure.com%2F.default"));
            assert!(body.contains(&format!("client_assertion={}", f.projected.lock().unwrap())));
            return Json(serde_json::json!({"access_token":"synthetic-access", "expires_in":1, "token_type":"Bearer"})).into_response();
        }
        f.calls.fetch_add(1, Ordering::SeqCst);
        assert_eq!(
            request.headers().get(header::AUTHORIZATION).unwrap(),
            "Bearer synthetic-access"
        );
        match f.mode.load(Ordering::SeqCst) {
            1 => {
                return (
                    StatusCode::TEMPORARY_REDIRECT,
                    [(header::LOCATION, "/redirected")],
                )
                    .into_response()
            }
            2 => return StatusCode::SERVICE_UNAVAILABLE.into_response(),
            3 => return StatusCode::UNAUTHORIZED.into_response(),
            4 => tokio::time::sleep(Duration::from_secs(30)).await,
            _ => (),
        }
        let key = request.uri().path().to_owned();
        if request.method() == axum::http::Method::PUT {
            assert_eq!(request.headers().get(header::IF_NONE_MATCH).unwrap(), "*");
            assert_eq!(
                request.headers().get("x-ms-blob-type").unwrap(),
                "BlockBlob"
            );
            let body = to_bytes(request.into_body(), CHUNK + 1).await.unwrap();
            let mut objects = f.objects.lock().unwrap();
            if objects.contains_key(&key) {
                return StatusCode::PRECONDITION_FAILED.into_response();
            }
            objects.insert(key, body);
            // Storage commits, but the response is lost/unsuccessful. The caller
            // cannot infer whether it is safe to repeat a generating action.
            if f.mode
                .compare_exchange(5, 0, Ordering::SeqCst, Ordering::SeqCst)
                .is_ok()
            {
                return StatusCode::SERVICE_UNAVAILABLE.into_response();
            }
            return (StatusCode::CREATED, [(header::ETAG, "\"immutable\"")]).into_response();
        }
        let bytes = f.objects.lock().unwrap().get(&key).cloned();
        match bytes {
            Some(bytes) => (
                [
                    (header::ETAG, "\"immutable\""),
                    (header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT"),
                ],
                bytes,
            )
                .into_response(),
            None => StatusCode::NOT_FOUND.into_response(),
        }
    }
    async fn serve(f: Fixture) -> (String, tokio::task::JoinHandle<()>) {
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let endpoint = format!("http://{}", listener.local_addr().unwrap());
        let task = tokio::spawn(async move {
            axum::serve(listener, Router::new().fallback(any(handle)).with_state(f))
                .await
                .unwrap()
        });
        (endpoint, task)
    }
    fn config() -> Config {
        Config {
            deployment: "ows-test".into(),
            account: "owteststorage".into(),
            container: "artifacts".into(),
            tenant_id: "11111111-1111-4111-8111-111111111111".into(),
            client_id: "22222222-2222-4222-8222-222222222222".into(),
            federated_token_file: "/nonexistent/azure-test-token".into(),
        }
    }
    fn fixture_store(config: &Config, endpoint: &str) -> Store {
        config
            .build_inner(
                endpoint,
                Some(Arc::new(StaticCredentialProvider::new(
                    AzureCredential::BearerToken("synthetic-access".into()),
                ))),
                true,
                None,
            )
            .unwrap()
    }
    fn artifact(store: &Store, bytes: &[u8]) -> Artifact {
        let id = format!("a_{}", uuid::Uuid::new_v4().simple());
        Artifact {
            id: id.clone(),
            registration: Registration {
                origin: Default::default(),
                request_id: "request".into(),
                session_id: "s_test".into(),
                turn_id: "t_test".into(),
                name: "sample.bin".into(),
                content_type: "application/octet-stream".into(),
                expected: Expected {
                    size: bytes.len() as u64,
                    sha256: hash(bytes),
                },
            },
            binding: store.binding().clone(),
            key: format!("{}/s_test/{id}", store.binding().deployment),
            status: Status::Pending,
            proof: None,
        }
    }
    fn assert_error<T: std::fmt::Debug>(result: Result<T>, expected: Error) {
        assert_eq!(result.unwrap_err().downcast_ref::<Error>(), Some(&expected));
    }
    #[tokio::test]
    async fn immutable_roundtrip_conflict_hash_scope_and_cancellation() {
        let fixture = Fixture::default();
        let (endpoint, server) = serve(fixture.clone()).await;
        let config = config();
        let store = fixture_store(&config, &endpoint);
        let bytes: Vec<_> = (0..CHUNK * 2 + 17).map(|i| (i % 251) as u8).collect();
        let a = artifact(&store, &bytes);
        let cancel = CancellationToken::new();
        let (one, two) = tokio::join!(
            store.put(&a, bytes.as_slice(), &cancel),
            store.put(&a, bytes.as_slice(), &cancel)
        );
        let proof = one.unwrap();
        assert_eq!(two.unwrap(), proof);
        assert_eq!(
            store
                .get(&a, &proof, cancel.clone())
                .await
                .unwrap()
                .try_collect::<Vec<_>>()
                .await
                .unwrap()
                .concat(),
            bytes
        );
        let mut wrong = bytes.clone();
        wrong[0] ^= 1;
        assert_error(
            store.put(&a, wrong.as_slice(), &cancel).await,
            Error::Conflict,
        );
        assert_eq!(store.verify(&a, &cancel).await.unwrap(), proof);
        for field in ["deployment", "account", "container", "client", "tenant"] {
            let mut other = config.clone();
            match field {
                "deployment" => other.deployment = "other".into(),
                "account" => other.account = "otheraccount".into(),
                "container" => other.container = "other-container".into(),
                "client" => other.client_id = "33333333-3333-4333-8333-333333333333".into(),
                _ => other.tenant_id = "44444444-4444-4444-8444-444444444444".into(),
            }
            let before = fixture.calls.load(Ordering::SeqCst);
            assert_error(
                fixture_store(&other, &endpoint).head(&a, &cancel).await,
                Error::Conflict,
            );
            assert_eq!(fixture.calls.load(Ordering::SeqCst), before);
        }
        let stopped = CancellationToken::new();
        stopped.cancel();
        assert_error(
            store.put(&a, bytes.as_slice(), &stopped).await,
            Error::Cancelled,
        );
        fixture.objects.lock().unwrap().insert(
            format!("/artifacts/{}/part-00", a.key),
            Bytes::from(vec![7; CHUNK]),
        );
        assert_error(store.verify(&a, &cancel).await, Error::Conflict);
        let empty = artifact(&store, b"");
        let proof = store.put(&empty, &b""[..], &cancel).await.unwrap();
        assert_eq!(store.verify(&empty, &cancel).await.unwrap(), proof);
        server.abort();
    }
    #[tokio::test]
    async fn bounded_reads_no_redirect_or_replay_and_inflight_cancel() {
        let fixture = Fixture::default();
        let (endpoint, server) = serve(fixture.clone()).await;
        let store = fixture_store(&config(), &endpoint);
        assert_error(
            store
                .backend
                .create("sample", Bytes::from(vec![0; CHUNK + 1]))
                .await,
            Error::TooLarge,
        );
        fixture
            .objects
            .lock()
            .unwrap()
            .insert("/artifacts/large".into(), Bytes::from(vec![0; 64]));
        assert_error(store.backend.read("large", 63).await, Error::Conflict);
        assert_error(
            store.backend.read("large", CHUNK + 1).await,
            Error::TooLarge,
        );
        assert_error(store.backend.read("missing", CHUNK).await, Error::NotFound);
        for mode in [1, 2, 3] {
            fixture.mode.store(mode, Ordering::SeqCst);
            let before = fixture.calls.load(Ordering::SeqCst);
            assert_error(
                store
                    .backend
                    .create("unknown-outcome", Bytes::from_static(b"data"))
                    .await,
                Error::Unavailable,
            );
            assert_eq!(
                fixture.calls.load(Ordering::SeqCst),
                before + 1,
                "redirect/auth/server failures must not replay writes"
            );
        }
        fixture.mode.store(5, Ordering::SeqCst);
        let interrupted = artifact(&store, b"committed-once");
        let before = fixture.calls.load(Ordering::SeqCst);
        assert_error(
            store
                .put(
                    &interrupted,
                    &b"committed-once"[..],
                    &CancellationToken::new(),
                )
                .await,
            Error::Unavailable,
        );
        assert_eq!(fixture.calls.load(Ordering::SeqCst), before + 1);
        // Explicit resubmission reconciles the existing chunk and publishes a
        // manifest. It does not regenerate bytes or overwrite stored objects.
        store
            .put(
                &interrupted,
                &b"committed-once"[..],
                &CancellationToken::new(),
            )
            .await
            .unwrap();
        fixture.mode.store(4, Ordering::SeqCst);
        let cancel = CancellationToken::new();
        let stop = cancel.clone();
        tokio::spawn(async move {
            tokio::time::sleep(Duration::from_millis(50)).await;
            stop.cancel()
        });
        let a = artifact(&store, b"data");
        assert_error(
            tokio::time::timeout(Duration::from_secs(1), store.put(&a, &b"data"[..], &cancel))
                .await
                .unwrap(),
            Error::Cancelled,
        );
        server.abort();
    }
    #[tokio::test]
    async fn workload_exchange_rereads_rotated_projected_token_without_fallback() {
        let fixture = Fixture::default();
        let (endpoint, server) = serve(fixture.clone()).await;
        let dir = tempfile::tempdir().unwrap();
        let mut config = config();
        config.federated_token_file = dir.path().join("token");
        std::fs::write(&config.federated_token_file, "projected-one").unwrap();
        *fixture.projected.lock().unwrap() = "projected-one".into();
        let store = config
            .build_inner(&endpoint, None, true, Some(&endpoint))
            .unwrap();
        store
            .backend
            .create("one", Bytes::from_static(b"1"))
            .await
            .unwrap();
        std::fs::write(&config.federated_token_file, "projected-two").unwrap();
        *fixture.projected.lock().unwrap() = "projected-two".into();
        tokio::time::sleep(Duration::from_millis(110)).await;
        store
            .backend
            .create("two", Bytes::from_static(b"2"))
            .await
            .unwrap();
        assert_eq!(fixture.tokens.load(Ordering::SeqCst), 2);
        std::fs::write(&config.federated_token_file, "projected-two").unwrap();
        for mode in [6, 7] {
            fixture.mode.store(mode, Ordering::SeqCst);
            let rejected = config
                .build_inner(&endpoint, None, true, Some(&endpoint))
                .unwrap();
            let before = fixture.tokens.load(Ordering::SeqCst);
            assert_error(
                rejected
                    .backend
                    .create("denied", Bytes::from_static(b"3"))
                    .await,
                Error::Unavailable,
            );
            assert_eq!(fixture.tokens.load(Ordering::SeqCst), before + 1);
            assert_eq!(
                fixture.calls.load(Ordering::SeqCst),
                2,
                "failed token exchange must not reach storage"
            );
        }
        std::fs::remove_file(&config.federated_token_file).unwrap();
        tokio::time::sleep(Duration::from_millis(110)).await;
        assert_error(
            store
                .backend
                .create("three", Bytes::from_static(b"3"))
                .await,
            Error::Unavailable,
        );
        assert_eq!(fixture.calls.load(Ordering::SeqCst), 2);
        assert_eq!(fixture.tokens.load(Ordering::SeqCst), 4);
        server.abort();
    }
    #[test]
    fn rejects_credentials_endpoints_and_invalid_scope_in_configuration() {
        let good = config();
        good.build().unwrap(); // Lazy: no token read or network during configuration.
        for key in [
            "endpoint",
            "access_key",
            "sas_token",
            "client_secret",
            "authority_host",
        ] {
            let mut value = serde_json::to_value(&good).unwrap();
            value[key] = "forbidden".into();
            assert!(serde_json::from_value::<Config>(value).is_err());
        }
        for (field, value) in [
            ("account", "account.example.com"),
            ("container", "bad--container"),
            ("tenant_id", "invalid"),
            ("client_id", "00000000-0000-0000-0000-000000000000"),
            ("federated_token_file", "relative"),
            ("federated_token_file", "/tmp/../token"),
        ] {
            let mut config = serde_json::to_value(&good).unwrap();
            config[field] = value.into();
            assert!(serde_json::from_value::<Config>(config)
                .unwrap()
                .build()
                .is_err());
        }
    }
}