Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat(storage): container credentials and allow_http for S3 stores (#10356)

Myles Scolnick7e9d1d846a6c
diff --git a/docs/guides/working_with_data/remote_storage.md b/docs/guides/working_with_data/remote_storage.md
index b0c291fcef..ba68132af2 100644
--- a/docs/guides/working_with_data/remote_storage.md
+++ b/docs/guides/working_with_data/remote_storage.md
@@ -49,6 +49,29 @@ store = S3Store.from_url(
)
```
+S3-compatible stores can also authenticate with container-vended credentials — used by ECS/EKS task roles and CoreWeave sandboxes, where the platform injects a credential endpoint and a token file:
+
+```python
+import os
+
+from obstore.store import S3Store
+
+store = S3Store(
+ "my-bucket",
+ endpoint="https://my-bucket.cwobject.com",
+ virtual_hosted_style_request=True,
+ container_credentials_full_uri=os.environ["AWS_CONTAINER_CREDENTIALS_FULL_URI"],
+ container_authorization_token_file=os.environ["AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE"],
+)
+```
+
+!!! note "S3 endpoint gotchas"
+
+ - `AWS_ENDPOINT_URL_S3` takes precedence over the `endpoint` argument, so an
+ explicitly configured endpoint is silently ignored when that variable is set.
+ - `virtual_hosted_style_request=True` expects the bucket name to already be part
+ of the endpoint hostname; unlike boto3, obstore does not prepend it.
+
#### fsspec
```python
diff --git a/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap b/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap
index 9cdd758d8c..2f3a823371 100644
--- a/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap
+++ b/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap
@@ -51,6 +51,21 @@ store = S3Store("operator-bucket",
)"
`;
+exports[`generateStorageCode > CoreWeave > with container credentials 1`] = `
+"from obstore.store import S3Store
+import os
+
+_container_credentials_full_uri = os.environ.get("AWS_CONTAINER_CREDENTIALS_FULL_URI")
+_container_authorization_token_file = os.environ.get("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE")
+store = S3Store("operator-bucket",
+ region="US-EAST-04A",
+ container_credentials_full_uri=_container_credentials_full_uri,
+ container_authorization_token_file=_container_authorization_token_file,
+ endpoint="https://operator-bucket.cwobject.com",
+ virtual_hosted_style_request=True,
+)"
+`;
+
exports[`generateStorageCode > CoreWeave > with secrets 1`] = `
"from obstore.store import S3Store
import os
@@ -119,6 +134,29 @@ exports[`generateStorageCode > S3 > minimal connection (bucket only) 1`] = `
store = S3Store("my-bucket",)"
`;
+exports[`generateStorageCode > S3 > with container credentials 1`] = `
+"from obstore.store import S3Store
+
+store = S3Store("my-bucket",
+ region="us-east-1",
+ container_credentials_full_uri="http://169.254.170.23/v1/credentials",
+ container_authorization_token_file="/var/run/secrets/token",
+)"
+`;
+
+exports[`generateStorageCode > S3 > with container credentials from secrets 1`] = `
+"from obstore.store import S3Store
+import os
+
+_container_credentials_full_uri = os.environ.get("AWS_CONTAINER_CREDENTIALS_FULL_URI")
+_container_authorization_token_file = os.environ.get("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE")
+store = S3Store("my-bucket",
+ region="us-east-1",
+ container_credentials_full_uri=_container_credentials_full_uri,
+ container_authorization_token_file=_container_authorization_token_file,
+)"
+`;
+
exports[`generateStorageCode > S3 > with custom endpoint 1`] = `
"from obstore.store import S3Store
@@ -130,6 +168,18 @@ store = S3Store("my-bucket",
)"
`;
+exports[`generateStorageCode > S3 > with plain-http endpoint 1`] = `
+"from obstore.store import S3Store
+
+store = S3Store("my-bucket",
+ region="us-east-1",
+ access_key_id="AKIAIOSFODNN7EXAMPLE",
+ secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ endpoint_url="http://cwlota.com",
+ allow_http=True,
+)"
+`;
+
exports[`generateStorageCode > S3 > with secrets 1`] = `
"from obstore.store import S3Store
import os
diff --git a/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts b/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts
index 44194ad4a9..16d13afcc4 100644
--- a/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts
+++ b/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts
@@ -9,9 +9,13 @@ describe("generateStorageCode", () => {
type: "s3",
bucket: "my-bucket",
region: "us-east-1",
- access_key_id: "AKIAIOSFODNN7EXAMPLE",
- secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
endpoint_url: undefined,
+ allow_http: false,
+ auth: {
+ type: "Access keys",
+ access_key_id: "AKIAIOSFODNN7EXAMPLE",
+ secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ },
};
const baseGCS: StorageConnection = {
@@ -31,8 +35,11 @@ describe("generateStorageCode", () => {
type: "coreweave",
bucket: "operator-bucket",
region: "US-EAST-04A",
- access_key_id: "AKIAIOSFODNN7EXAMPLE",
- secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ auth: {
+ type: "Access keys",
+ access_key_id: "AKIAIOSFODNN7EXAMPLE",
+ secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ },
};
const baseGDrive: StorageConnection = {
@@ -52,6 +59,8 @@ describe("generateStorageCode", () => {
const conn: StorageConnection = {
type: "s3",
bucket: "my-bucket",
+ allow_http: false,
+ auth: { type: "Access keys" },
};
expect(
generateStorageCode(conn, { library: "obstore" }),
@@ -68,13 +77,61 @@ describe("generateStorageCode", () => {
).toMatchSnapshot();
});
+ it("with plain-http endpoint", () => {
+ const conn: StorageConnection = {
+ ...baseS3,
+ endpoint_url: "http://cwlota.com",
+ allow_http: true,
+ };
+ expect(
+ generateStorageCode(conn, { library: "obstore" }),
+ ).toMatchSnapshot();
+ });
+
+ it("with container credentials", () => {
+ const conn: StorageConnection = {
+ ...baseS3,
+ auth: {
+ type: "Container credentials",
+ container_credentials_full_uri:
+ "http://169.254.170.23/v1/credentials",
+ container_authorization_token_file: "/var/run/secrets/token",
+ },
+ };
+ expect(
+ generateStorageCode(conn, { library: "obstore" }),
+ ).toMatchSnapshot();
+ });
+
+ it("with container credentials from secrets", () => {
+ const conn: StorageConnection = {
+ ...baseS3,
+ auth: {
+ type: "Container credentials",
+ container_credentials_full_uri: prefixSecret(
+ "AWS_CONTAINER_CREDENTIALS_FULL_URI",
+ ),
+ container_authorization_token_file: prefixSecret(
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
+ ),
+ },
+ };
+ expect(
+ generateStorageCode(conn, { library: "obstore" }),
+ ).toMatchSnapshot();
+ });
+
it("with secrets", () => {
const conn: StorageConnection = {
type: "s3",
bucket: "my-bucket",
region: "us-east-1",
- access_key_id: prefixSecret("AWS_ACCESS_KEY_ID"),
- secret_access_key: prefixSecret("AWS_SECRET_ACCESS_KEY"),
+ allow_http: false,
+ auth: {
+ type: "Access keys",
+ access_key_id: prefixSecret("AWS_ACCESS_KEY_ID"),
+ secret_access_key: prefixSecret("AWS_SECRET_ACCESS_KEY"),
+ },
};
expect(
generateStorageCode(conn, { library: "obstore" }),
@@ -143,6 +200,25 @@ describe("generateStorageCode", () => {
type: "coreweave",
bucket: "operator-bucket",
region: "US-EAST-04A",
+ auth: { type: "Access keys" },
+ };
+ expect(
+ generateStorageCode(conn, { library: "obstore" }),
+ ).toMatchSnapshot();
+ });
+
+ it("with container credentials", () => {
+ const conn: StorageConnection = {
+ ...baseCoreWeave,
+ auth: {
+ type: "Container credentials",
+ container_credentials_full_uri: prefixSecret(
+ "AWS_CONTAINER_CREDENTIALS_FULL_URI",
+ ),
+ container_authorization_token_file: prefixSecret(
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
+ ),
+ },
};
expect(
generateStorageCode(conn, { library: "obstore" }),
@@ -154,8 +230,11 @@ describe("generateStorageCode", () => {
type: "coreweave",
bucket: "operator-bucket",
region: "US-EAST-04A",
- access_key_id: prefixSecret("COREWEAVE_OBJECT_STORAGE_KEY"),
- secret_access_key: prefixSecret("COREWEAVE_OBJECT_STORAGE_SECRET"),
+ auth: {
+ type: "Access keys",
+ access_key_id: prefixSecret("COREWEAVE_OBJECT_STORAGE_KEY"),
+ secret_access_key: prefixSecret("COREWEAVE_OBJECT_STORAGE_SECRET"),
+ },
};
expect(
generateStorageCode(conn, { library: "obstore" }),
diff --git a/frontend/src/components/editor/connections/storage/as-code.ts b/frontend/src/components/editor/connections/storage/as-code.ts
index 142f79158d..ff8071789c 100644
--- a/frontend/src/components/editor/connections/storage/as-code.ts
+++ b/frontend/src/components/editor/connections/storage/as-code.ts
@@ -3,7 +3,11 @@
import dedent from "string-dedent";
import { assertNever } from "@/utils/assertNever";
import { isSecret, unprefixSecret } from "../secrets";
-import { type StorageConnection, StorageConnectionSchema } from "./schemas";
+import {
+ type S3Auth,
+ type StorageConnection,
+ StorageConnectionSchema,
+} from "./schemas";
export type StorageLibrary = "obstore" | "fsspec";
@@ -50,6 +54,36 @@ class SecretContainer {
}
}
+/**
+ * Credential kwargs shared by every S3-compatible store.
+ */
+function s3AuthParams(
+ auth: S3Auth | undefined,
+ secrets: SecretContainer,
+): string[] {
+ // obstore polls the credential endpoint with the token read from the file,
+ // refreshing credentials for the lifetime of the store.
+ if (auth?.type === "Container credentials") {
+ return [
+ ` container_credentials_full_uri=${secrets.print("container_credentials_full_uri", auth.container_credentials_full_uri)},`,
+ ` container_authorization_token_file=${secrets.print("container_authorization_token_file", auth.container_authorization_token_file)},`,
+ ];
+ }
+
+ const params: string[] = [];
+ if (auth?.access_key_id) {
+ params.push(
+ ` access_key_id=${secrets.print("access_key_id", auth.access_key_id)},`,
+ );
+ }
+ if (auth?.secret_access_key) {
+ params.push(
+ ` secret_access_key=${secrets.print("secret_access_key", auth.secret_access_key)},`,
+ );
+ }
+ return params;
+}
+
function generateS3Code(
connection: Extract<StorageConnection, { type: "s3" }>,
secrets: SecretContainer,
@@ -61,21 +95,15 @@ function generateS3Code(
if (connection.region) {
params.push(` region=${secrets.print("region", connection.region)},`);
}
- if (connection.access_key_id) {
- params.push(
- ` access_key_id=${secrets.print("access_key_id", connection.access_key_id)},`,
- );
- }
- if (connection.secret_access_key) {
- params.push(
- ` secret_access_key=${secrets.print("secret_access_key", connection.secret_access_key)},`,
- );
- }
+ params.push(...s3AuthParams(connection.auth, secrets));
if (connection.endpoint_url) {
params.push(
` endpoint_url=${secrets.print("endpoint_url", connection.endpoint_url)},`,
);
}
+ if (connection.allow_http) {
+ params.push(" allow_http=True,");
+ }
const paramsStr = params.length > 0 ? `\n${params.join("\n")}\n` : "";
@@ -140,16 +168,7 @@ function generateCoreWeaveCode(
` region=${secrets.print("region", connection.region)},`,
];
- if (connection.access_key_id) {
- params.push(
- ` access_key_id=${secrets.print("access_key_id", connection.access_key_id)},`,
- );
- }
- if (connection.secret_access_key) {
- params.push(
- ` secret_access_key=${secrets.print("secret_access_key", connection.secret_access_key)},`,
- );
- }
+ params.push(...s3AuthParams(connection.auth, secrets));
params.push(
` endpoint="https://${connection.bucket}.cwobject.com",`,
diff --git a/frontend/src/components/editor/connections/storage/schemas.ts b/frontend/src/components/editor/connections/storage/schemas.ts
index 1f7f0ca00a..5ca2f18565 100644
--- a/frontend/src/components/editor/connections/storage/schemas.ts
+++ b/frontend/src/components/editor/connections/storage/schemas.ts
@@ -2,6 +2,83 @@
import { z } from "zod";
import { FieldOptions } from "@/components/forms/options";
+/**
+ * Credentials shared by every S3-compatible store (Amazon S3, CoreWeave, ...).
+ *
+ * Rendered as tabs: either long-lived access keys, or a credential-vending
+ * endpoint (ECS/EKS task roles, CoreWeave sandboxes, ...) that obstore polls
+ * with a bearer token read from disk.
+ */
+const s3AuthField = () =>
+ z
+ .discriminatedUnion("type", [
+ z.object({
+ type: z.literal("Access keys"),
+ access_key_id: z
+ .string()
+ .optional()
+ .describe(
+ FieldOptions.of({
+ label: "Access Key ID",
+ description: "Leave empty to use the default credential chain",
+ inputType: "password",
+ optionRegex:
+ "(access.?key.?id|object.?storage.?key|aws.?access.?key)",
+ }),
+ ),
+ secret_access_key: z
+ .string()
+ .optional()
+ .describe(
+ FieldOptions.of({
+ label: "Secret Access Key",
+ inputType: "password",
+ optionRegex:
+ "(secret.?access.?key|object.?storage.?secret|aws.?secret)",
+ }),
+ ),
+ }),
+ z.object({
+ type: z.literal("Container credentials"),
+ container_credentials_full_uri: z
+ .string()
+ .nonempty()
+ .describe(
+ FieldOptions.of({
+ label: "Credentials URI",
+ description: "Endpoint that vends credentials to the container",
+ placeholder: "http://169.254.170.23/v1/credentials",
+ optionRegex: "container.?credentials",
+ }),
+ ),
+ container_authorization_token_file: z
+ .string()
+ .nonempty()
+ .describe(
+ FieldOptions.of({
+ label: "Authorization Token File",
+ description: "File holding the token sent to the credentials URI",
+ placeholder: "/var/run/secrets/token",
+ optionRegex:
+ "(container.?authorization.?token.?file|container.?auth.?token)",
+ }),
+ ),
+ }),
+ ])
+ .default({ type: "Access keys" })
+ .describe(FieldOptions.of({ label: "Credentials", special: "tabs" }));
+
+const allowHttpField = () =>
+ z
+ .boolean()
+ .default(false)
+ .describe(
+ FieldOptions.of({
+ label: "Allow HTTP",
+ description: "Required for plain-http (non-TLS) endpoints",
+ }),
+ );
+
export const S3StorageSchema = z
.object({
type: z.literal("s3"),
@@ -25,36 +102,20 @@ export const S3StorageSchema = z
optionRegex: "(region|aws.?region)",
}),
),
- access_key_id: z
- .string()
- .optional()
- .describe(
- FieldOptions.of({
- label: "Access Key ID",
- inputType: "password",
- optionRegex: "(access.?key.?id|aws.?access.?key)",
- }),
- ),
- secret_access_key: z
- .string()
- .optional()
- .describe(
- FieldOptions.of({
- label: "Secret Access Key",
- inputType: "password",
- optionRegex: "(secret.?access.?key|aws.?secret)",
- }),
- ),
endpoint_url: z
.string()
.optional()
.describe(
FieldOptions.of({
label: "Endpoint URL",
+ description:
+ "Ignored if the AWS_ENDPOINT_URL_S3 environment variable is set",
placeholder: "https://s3.amazonaws.com",
optionRegex: "(endpoint|s3.?url|s3.?endpoint)",
}),
),
+ allow_http: allowHttpField(),
+ auth: s3AuthField(),
})
.describe(FieldOptions.of({ direction: "two-columns" }));
@@ -139,28 +200,7 @@ export const CoreWeaveStorageSchema = z
placeholder: "US-EAST-04A",
}),
),
- access_key_id: z
- .string()
- .optional()
- .describe(
- FieldOptions.of({
- label: "Access Key ID",
- inputType: "password",
- optionRegex:
- "(access.?key.?id|object.?storage.?key|aws.?access.?key)",
- }),
- ),
- secret_access_key: z
- .string()
- .optional()
- .describe(
- FieldOptions.of({
- label: "Secret Access Key",
- inputType: "password",
- optionRegex:
- "(secret.?access.?key|object.?storage.?secret|aws.?secret)",
- }),
- ),
+ auth: s3AuthField(),
})
.describe(FieldOptions.of({ direction: "two-columns" }));
@@ -189,3 +229,4 @@ export const StorageConnectionSchema = z.discriminatedUnion("type", [
]);
export type StorageConnection = z.infer<typeof StorageConnectionSchema>;
+export type S3Auth = Extract<StorageConnection, { type: "s3" }>["auth"];
diff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.py
index bc8e7cdf2e..c223fd49ff 100644
--- a/marimo/_data/_external_storage/storage.py
+++ b/marimo/_data/_external_storage/storage.py
@@ -3,7 +3,9 @@ from __future__ import annotations
import asyncio
import mimetypes
+import re
from datetime import timedelta
+from functools import cached_property
from typing import TYPE_CHECKING, Any, Literal, cast
from marimo import _loggers
@@ -28,12 +30,9 @@ if TYPE_CHECKING:
from obstore import ObjectMeta
from obstore.store import (
AzureConfig,
- AzureStore,
GCSConfig,
- GCSStore,
ObjectStore, # noqa: F401
S3Config,
- S3Store,
)
LOGGER = _loggers.marimo_logger()
@@ -147,17 +146,31 @@ class Obstore(StorageBackend["ObjectStore"]):
LOGGER.info("Failed to sign URL for %s", path)
return None
- def _get_config(
- self, store: AzureStore | GCSStore | S3Store
- ) -> AzureConfig | GCSConfig | S3Config | None:
+ @cached_property
+ def _config(self) -> AzureConfig | GCSConfig | S3Config | None:
+ """The store's config, or None for stores that don't have one.
+
+ obstore (0.11.0) panics rather than raises for perfectly valid stores:
+ an `S3Store` created with `allow_http=True` fails with "Expected config
+ prefix to start with aws_" (pyo3-object_store/src/aws/store.rs:254),
+ printing raw Rust output to stderr. The result is cached so such a
+ store panics once, rather than on every property read.
+ """
+ from obstore.store import AzureStore, GCSStore, S3Store
+
+ store = self.store
+ if not isinstance(store, (AzureStore, GCSStore, S3Store)):
+ return None
try:
return store.config
- except BaseException:
- # Sometimes, there will be a Rust panic when trying to get the config for invalid stores
- LOGGER.exception(
- "Failed to read store config for %s", type(store).__name__
+ except BaseException as e:
+ # PanicException derives from BaseException, not Exception
+ LOGGER.warning(
+ "Failed to read store config for %s: %s",
+ type(store).__name__,
+ e,
)
- return None
+ return None
@property
def protocol(self) -> KNOWN_STORAGE_TYPES | str:
@@ -170,17 +183,14 @@ class Obstore(StorageBackend["ObjectStore"]):
S3Store,
)
- # Try the endpoint URL which can give a more accurate protocol
- if not isinstance(self.store, (MemoryStore, HTTPStore, LocalStore)):
- config = self._get_config(self.store)
- if config is None:
- return "unknown"
-
- endpoint = config.get("endpoint")
- if isinstance(endpoint, str) and (
- protocol := detect_protocol_from_url(endpoint)
- ):
- return protocol
+ # Try the endpoint URL which can give a more accurate protocol,
+ # falling back to the store type if the config is unreadable.
+ config = self._config
+ endpoint = config.get("endpoint") if config else None
+ if isinstance(endpoint, str) and (
+ protocol := detect_protocol_from_url(endpoint)
+ ):
+ return protocol
if isinstance(self.store, MemoryStore):
return "in-memory"
@@ -216,11 +226,13 @@ class Obstore(StorageBackend["ObjectStore"]):
if isinstance(self.store, LocalStore):
return None # root
- config = self._get_config(self.store)
+ config = self._config
if config is None:
- return None
+ # The config can be unreadable (see _config); the repr still
+ # carries the bucket/container name.
+ return _bucket_from_repr(self.store)
- bucket = config.get("bucket")
+ bucket = config.get("bucket") or config.get("container_name")
if bucket is None:
LOGGER.debug(
"No bucket found for storage backend. Config %s", config
@@ -500,6 +512,16 @@ _URL_PATTERNS: list[tuple[str, CLOUD_STORAGE_TYPES]] = [
]
+# e.g. 'S3Store(bucket="my-bucket")', 'AzureStore(container_name="c", ...)'
+_STORE_REPR_NAME_RE = re.compile(r'(?:bucket|container_name)="([^"]*)"')
+
+
+def _bucket_from_repr(store: object) -> str | None:
+ """Best-effort bucket/container name for a store with an unreadable config."""
+ match = _STORE_REPR_NAME_RE.search(repr(store))
+ return match.group(1) if match else None
+
+
def detect_protocol_from_url(url: str) -> CLOUD_STORAGE_TYPES | None:
"""Detect the storage provider from an endpoint URL."""
url = url.strip().lower()
diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.py
index f2612b7975..3849cf576b 100644
--- a/tests/_data/_external_storage/test_storage_models.py
+++ b/tests/_data/_external_storage/test_storage_models.py
@@ -513,6 +513,15 @@ class TestObstore:
root = backend.root_path
assert root == "test-bucket"
+ def test_root_path_azure_uses_container_name(self) -> None:
+ from obstore.store import AzureStore
+
+ store = AzureStore(
+ "my-container", account_name="acct", account_key="YWJjZA=="
+ )
+ backend = self._make_backend(store)
+ assert backend.root_path == "my-container"
+
def test_is_compatible_with_obstore(self) -> None:
from obstore.store import MemoryStore
@@ -588,37 +597,74 @@ class TestObstore:
backend = self._make_backend(store)
assert backend.display_name == "In-memory"
- def test_protocol_returns_unknown_when_config_panics(self) -> None:
+ def test_protocol_falls_back_to_store_type_when_config_panics(
+ self,
+ ) -> None:
from obstore.store import S3Store
store = S3Store("bucket", skip_signature=True)
backend = self._make_backend(store)
+ with self._panicking_config(store):
+ assert backend.protocol == "s3"
+
+ def test_root_path_falls_back_to_repr_when_config_panics(self) -> None:
+ from obstore.store import S3Store
+
+ store = S3Store("bucket", skip_signature=True)
+ backend = self._make_backend(store)
+
+ with self._panicking_config(store):
+ assert backend.root_path == "bucket"
+
+ def test_config_is_only_read_once_when_it_panics(self) -> None:
+ from obstore.store import S3Store
+
+ store = S3Store("bucket", skip_signature=True)
+ backend = self._make_backend(store)
+ calls = 0
+
def _raise(_: Any) -> None:
+ nonlocal calls
+ calls += 1
raise BaseException("rust panic") # noqa: TRY002
with patch.object(
- type(store),
- "config",
- new_callable=lambda: property(_raise),
+ type(store), "config", new_callable=lambda: property(_raise)
):
- assert backend.protocol == "unknown"
+ assert backend.protocol == "s3"
+ assert backend.root_path == "bucket"
+
+ assert calls == 1
- def test_root_path_returns_none_when_config_panics(self) -> None:
+ def test_container_credentials_store_is_readable(self) -> None:
+ """obstore panics on `store.config` for container-credential stores."""
from obstore.store import S3Store
- store = S3Store("bucket", skip_signature=True)
+ store = S3Store(
+ "demo-bucket",
+ endpoint="https://demo-bucket.cwobject.com",
+ allow_http=True,
+ virtual_hosted_style_request=True,
+ container_credentials_full_uri="http://169.254.170.23/v1/creds",
+ container_authorization_token_file="/tmp/token",
+ )
backend = self._make_backend(store)
+ assert backend.root_path == "demo-bucket"
+ # `allow_http=True` makes the config (and so the endpoint) unreadable,
+ # so we can't detect "coreweave" from the endpoint and fall back to the
+ # store type.
+ assert backend.protocol == "s3"
+
+ @staticmethod
+ def _panicking_config(store: Any) -> Any:
def _raise(_: Any) -> None:
raise BaseException("rust panic") # noqa: TRY002
- with patch.object(
- type(store),
- "config",
- new_callable=lambda: property(_raise),
- ):
- assert backend.root_path is None
+ return patch.object(
+ type(store), "config", new_callable=lambda: property(_raise)
+ )
@pytest.mark.skipif(not HAS_FSSPEC, reason="fsspec not installed")