Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

improve storage protocol detection (#8513)

Shahmir Varqhaad9a0d6717a2
diff --git a/frontend/src/components/storage/components.tsx b/frontend/src/components/storage/components.tsx
index 24e0644364..a8576ae156 100644
--- a/frontend/src/components/storage/components.tsx
+++ b/frontend/src/components/storage/components.tsx
@@ -13,6 +13,7 @@ import {
FileSpreadsheetIcon,
FileTextIcon,
FileVideoIcon,
+ GithubIcon,
GlobeIcon,
HardDriveIcon,
ImageIcon,
@@ -66,6 +67,7 @@ const PROTOCOL_ICONS: Record<KnownStorageProtocol, IconEntry> = {
http: GlobeIcon,
file: HardDriveIcon,
"in-memory": DatabaseZapIcon,
+ github: GithubIcon,
};
export const ProtocolIcon: React.FC<{
diff --git a/frontend/src/core/storage/types.ts b/frontend/src/core/storage/types.ts
index ab6d1545c2..c1c8230588 100644
--- a/frontend/src/core/storage/types.ts
+++ b/frontend/src/core/storage/types.ts
@@ -54,4 +54,5 @@ export type KnownStorageProtocol =
| "cloudflare"
| "http"
| "file"
- | "in-memory";
+ | "in-memory"
+ | "github";
diff --git a/marimo/_data/_external_storage/models.py b/marimo/_data/_external_storage/models.py
index 7c3b018409..c54624533d 100644
--- a/marimo/_data/_external_storage/models.py
+++ b/marimo/_data/_external_storage/models.py
@@ -3,16 +3,15 @@ from __future__ import annotations
import abc
from dataclasses import dataclass
-from typing import Any, Generic, Literal, TypeVar, get_args
+from typing import Any, Generic, Literal, TypeVar, cast, get_args
import msgspec
from marimo._types.ids import VariableName
from marimo._utils.assert_never import log_never
-KNOWN_STORAGE_TYPES = Literal[
- "s3", "gcs", "azure", "http", "file", "in-memory"
-]
+CLOUD_STORAGE_TYPES = Literal["s3", "gcs", "azure", "cloudflare", "coreweave"]
+KNOWN_STORAGE_TYPES = Literal[CLOUD_STORAGE_TYPES, "http", "file", "in-memory"]
SIGNED_URL_EXPIRATION = 60
@@ -140,21 +139,26 @@ class StorageBackend(abc.ABC, Generic[Backend]):
protocol = self.protocol
if protocol not in get_args(KNOWN_STORAGE_TYPES):
return protocol.capitalize()
- if protocol == "s3":
+ known = cast(KNOWN_STORAGE_TYPES, protocol)
+ if known == "s3":
return "Amazon S3"
- elif protocol == "gcs":
+ elif known == "gcs":
return "Google Cloud Storage"
- elif protocol == "azure":
+ elif known == "azure":
return "Azure Blob Storage"
- elif protocol == "http":
+ elif known == "cloudflare":
+ return "Cloudflare R2"
+ elif known == "http":
return "HTTP"
- elif protocol == "file":
+ elif known == "file":
return "File"
- elif protocol == "in-memory":
+ elif known == "in-memory":
return "In-memory"
+ elif known == "coreweave":
+ return "Coreweave"
else:
- log_never(protocol) # type: ignore[arg-type]
- return protocol
+ log_never(known)
+ return known
@property
@abc.abstractmethod
diff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.py
index 52b5757847..c08e1d3b2d 100644
--- a/marimo/_data/_external_storage/storage.py
+++ b/marimo/_data/_external_storage/storage.py
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast
from marimo import _loggers
from marimo._data._external_storage.models import (
+ CLOUD_STORAGE_TYPES,
DEFAULT_FETCH_LIMIT,
KNOWN_STORAGE_TYPES,
SIGNED_URL_EXPIRATION,
@@ -139,6 +140,14 @@ class Obstore(StorageBackend["ObjectStore"]):
S3Store,
)
+ # Try the endpoint URL which can give a more accurate protocol
+ if not isinstance(self.store, (MemoryStore, HTTPStore, LocalStore)):
+ endpoint = self.store.config.get("endpoint")
+ if isinstance(endpoint, str) and (
+ protocol := detect_protocol_from_url(endpoint)
+ ):
+ return protocol
+
if isinstance(self.store, MemoryStore):
return "in-memory"
elif isinstance(self.store, HTTPStore):
@@ -317,9 +326,23 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):
@property
def protocol(self) -> KNOWN_STORAGE_TYPES | str:
- if isinstance(self.store.protocol, tuple):
- return normalize_protocol("-".join(self.store.protocol))
- return normalize_protocol(self.store.protocol)
+ store_protocol = self.store.protocol
+ storage_options = self.store.storage_options
+
+ # Try the endpoint URL which can give a more accurate protocol
+ endpoint_url = storage_options.get("endpoint_url")
+ if isinstance(endpoint_url, str) and (
+ protocol := detect_protocol_from_url(endpoint_url)
+ ):
+ return protocol
+
+ if isinstance(store_protocol, tuple):
+ for store_protocol_item in store_protocol:
+ if normalized := normalize_protocol(store_protocol_item):
+ return normalized
+ return "-".join(store_protocol)
+
+ return normalize_protocol(store_protocol) or store_protocol
@property
def root_path(self) -> str | None:
@@ -335,18 +358,50 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):
return isinstance(var, AbstractFileSystem)
-def normalize_protocol(protocol: str) -> KNOWN_STORAGE_TYPES | str:
- """Normalize the protocol to a known storage type."""
- protocol = protocol.strip().lower()
- if "s3" in protocol:
- return "s3"
- elif "gcs" in protocol or "gs" in protocol:
- return "gcs"
- elif "azure" in protocol:
- return "azure"
- elif "http" in protocol:
- return "http"
- elif "file" in protocol:
- return "file"
- else:
- return protocol
+_PROTOCOL_MAP: dict[str, KNOWN_STORAGE_TYPES] = {
+ "s3": "s3",
+ "s3a": "s3",
+ "gs": "gcs",
+ "gcs": "gcs",
+ "abfs": "azure",
+ "abfss": "azure",
+ "az": "azure",
+ "adl": "azure",
+ "http": "http",
+ "https": "http",
+ "file": "file",
+ "local": "file",
+ "memory": "in-memory",
+ "r2": "cloudflare",
+}
+
+# Specific provider patterns checked before generic ones (e.g. S3),
+# since S3-compatible services may also have "s3" in their URL.
+# The order of the patterns is important, the first pattern that matches will be used.
+_URL_PATTERNS: list[tuple[str, CLOUD_STORAGE_TYPES]] = [
+ ("cloudflare", "cloudflare"),
+ ("r2.", "cloudflare"),
+ ("cwobject", "coreweave"),
+ ("cwlota", "coreweave"),
+ ("coreweave", "coreweave"),
+ ("blob.core.windows", "azure"),
+ ("azure", "azure"),
+ ("googleapis", "gcs"),
+ ("storage.google", "gcs"),
+ ("s3", "s3"),
+ ("amazonaws", "s3"),
+]
+
+
+def detect_protocol_from_url(url: str) -> CLOUD_STORAGE_TYPES | None:
+ """Detect the storage provider from an endpoint URL."""
+ url = url.strip().lower()
+ for pattern, protocol in _URL_PATTERNS:
+ if pattern in url:
+ return protocol
+ return None
+
+
+def normalize_protocol(protocol: str) -> KNOWN_STORAGE_TYPES | None:
+ """Normalize a protocol string (e.g. 's3a', 'gs', 'abfs') to a known storage type."""
+ return _PROTOCOL_MAP.get(protocol.strip().lower())
diff --git a/tests/_data/_external_storage/test_get_storage.py b/tests/_data/_external_storage/test_get_storage.py
index 350a2e9735..1bf597b402 100644
--- a/tests/_data/_external_storage/test_get_storage.py
+++ b/tests/_data/_external_storage/test_get_storage.py
@@ -152,8 +152,8 @@ class TestStorageBackendToStorageNamespace:
assert result == snapshot(
StorageNamespace(
name=VariableName("mem_fs"),
- display_name="Memory",
- protocol="memory",
+ display_name="In-memory",
+ protocol="in-memory",
root_path="/",
storage_entries=[],
)
diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.py
index e3f79726d8..09c3971c02 100644
--- a/tests/_data/_external_storage/test_storage_models.py
+++ b/tests/_data/_external_storage/test_storage_models.py
@@ -14,6 +14,7 @@ from marimo._data._external_storage.models import DownloadResult, StorageEntry
from marimo._data._external_storage.storage import (
FsspecFilesystem,
Obstore,
+ detect_protocol_from_url,
normalize_protocol,
)
from marimo._dependencies.dependencies import DependencyManager
@@ -1024,8 +1025,7 @@ class TestFsspecFilesystemIntegration:
fs = MemoryFileSystem()
backend = FsspecFilesystem(fs, VariableName("mem_fs"))
- # MemoryFileSystem protocol is "memory", which doesn't match known types
- assert backend.protocol == "memory"
+ assert backend.protocol == "in-memory"
@pytest.mark.skipif(not HAS_OBSTORE, reason="obstore not installed")
@@ -1152,28 +1152,51 @@ class TestNormalizeProtocol:
("s3", "s3"),
("s3a", "s3"),
("S3", "s3"),
+ ("gs", "gcs"),
("gcs", "gcs"),
- ("GCS", "gcs"),
- ("gs-gcs", "gcs"),
- ("azure", "azure"),
- ("Azure", "azure"),
- ("abfs-azure", "azure"),
+ ("abfs", "azure"),
+ ("abfss", "azure"),
+ ("az", "azure"),
("http", "http"),
- ("HTTP", "http"),
- ("https-http", "http"),
+ ("https", "http"),
("file", "file"),
- ("FILE", "file"),
- ("local-file", "file"),
- ("ftp", "ftp"),
- ("custom", "custom"),
+ ("local", "file"),
+ ("memory", "in-memory"),
+ ("r2", "cloudflare"),
(" s3 ", "s3"),
- (" gcs ", "gcs"),
+ ("unknown", None),
+ ("ftp", None),
],
)
- def test_normalize_protocol(self, protocol: str, expected: str) -> None:
+ def test_normalize_protocol(
+ self, protocol: str, expected: str | None
+ ) -> None:
assert normalize_protocol(protocol) == expected
+class TestDetectProtocolFromUrl:
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
+ ("https://account.r2.cloudflarestorage.com", "cloudflare"),
+ ("https://s3.amazonaws.com", "s3"),
+ ("https://s3.us-east-1.amazonaws.com", "s3"),
+ ("https://storage.googleapis.com", "gcs"),
+ ("https://account.blob.core.windows.net", "azure"),
+ ("https://minio.example.com", None),
+ ("https://my-custom-endpoint.com", None),
+ (
+ "https://s3.cloudflare.com",
+ "cloudflare",
+ ), # Although there is S3, it will match the cloudflare pattern first
+ ],
+ )
+ def test_detect_protocol_from_url(
+ self, url: str, expected: str | None
+ ) -> None:
+ assert detect_protocol_from_url(url) == expected
+
+
# --- Helpers ---