A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
improve storage protocol detection (#8513)
Shahmir Varqhaad9a0d6717a2
diff --git a/frontend/src/components/storage/components.tsx b/frontend/src/components/storage/components.tsxindex 24e0644364..a8576ae156 100644--- a/frontend/src/components/storage/components.tsx+++ b/frontend/src/components/storage/components.tsx@@ -13,6 +13,7 @@ import {FileSpreadsheetIcon,FileTextIcon,FileVideoIcon,+ GithubIcon,GlobeIcon,HardDriveIcon,ImageIcon,@@ -66,6 +67,7 @@ const PROTOCOL_ICONS: Record<KnownStorageProtocol, IconEntry> = {http: GlobeIcon,file: HardDriveIcon,"in-memory": DatabaseZapIcon,+ github: GithubIcon,};export const ProtocolIcon: React.FC<{diff --git a/frontend/src/core/storage/types.ts b/frontend/src/core/storage/types.tsindex ab6d1545c2..c1c8230588 100644--- a/frontend/src/core/storage/types.ts+++ b/frontend/src/core/storage/types.ts@@ -54,4 +54,5 @@ export type KnownStorageProtocol =| "cloudflare"| "http"| "file"- | "in-memory";+ | "in-memory"+ | "github";diff --git a/marimo/_data/_external_storage/models.py b/marimo/_data/_external_storage/models.pyindex 7c3b018409..c54624533d 100644--- a/marimo/_data/_external_storage/models.py+++ b/marimo/_data/_external_storage/models.py@@ -3,16 +3,15 @@ from __future__ import annotationsimport abcfrom dataclasses import dataclass-from typing import Any, Generic, Literal, TypeVar, get_args+from typing import Any, Generic, Literal, TypeVar, cast, get_argsimport msgspecfrom marimo._types.ids import VariableNamefrom marimo._utils.assert_never import log_never-KNOWN_STORAGE_TYPES = Literal[- "s3", "gcs", "azure", "http", "file", "in-memory"-]+CLOUD_STORAGE_TYPES = Literal["s3", "gcs", "azure", "cloudflare", "coreweave"]+KNOWN_STORAGE_TYPES = Literal[CLOUD_STORAGE_TYPES, "http", "file", "in-memory"]SIGNED_URL_EXPIRATION = 60@@ -140,21 +139,26 @@ class StorageBackend(abc.ABC, Generic[Backend]):protocol = self.protocolif protocol not in get_args(KNOWN_STORAGE_TYPES):return protocol.capitalize()- if protocol == "s3":+ known = cast(KNOWN_STORAGE_TYPES, protocol)+ if known == "s3":return "Amazon S3"- elif protocol == "gcs":+ elif known == "gcs":return "Google Cloud Storage"- elif protocol == "azure":+ elif known == "azure":return "Azure Blob Storage"- elif protocol == "http":+ elif known == "cloudflare":+ return "Cloudflare R2"+ elif known == "http":return "HTTP"- elif protocol == "file":+ elif known == "file":return "File"- elif protocol == "in-memory":+ elif known == "in-memory":return "In-memory"+ elif known == "coreweave":+ return "Coreweave"else:- log_never(protocol) # type: ignore[arg-type]- return protocol+ log_never(known)+ return known@property@abc.abstractmethoddiff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.pyindex 52b5757847..c08e1d3b2d 100644--- a/marimo/_data/_external_storage/storage.py+++ b/marimo/_data/_external_storage/storage.py@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Literal, castfrom marimo import _loggersfrom marimo._data._external_storage.models import (+ CLOUD_STORAGE_TYPES,DEFAULT_FETCH_LIMIT,KNOWN_STORAGE_TYPES,SIGNED_URL_EXPIRATION,@@ -139,6 +140,14 @@ class Obstore(StorageBackend["ObjectStore"]):S3Store,)+ # Try the endpoint URL which can give a more accurate protocol+ if not isinstance(self.store, (MemoryStore, HTTPStore, LocalStore)):+ endpoint = self.store.config.get("endpoint")+ if isinstance(endpoint, str) and (+ protocol := detect_protocol_from_url(endpoint)+ ):+ return protocol+if isinstance(self.store, MemoryStore):return "in-memory"elif isinstance(self.store, HTTPStore):@@ -317,9 +326,23 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):@propertydef protocol(self) -> KNOWN_STORAGE_TYPES | str:- if isinstance(self.store.protocol, tuple):- return normalize_protocol("-".join(self.store.protocol))- return normalize_protocol(self.store.protocol)+ store_protocol = self.store.protocol+ storage_options = self.store.storage_options++ # Try the endpoint URL which can give a more accurate protocol+ endpoint_url = storage_options.get("endpoint_url")+ if isinstance(endpoint_url, str) and (+ protocol := detect_protocol_from_url(endpoint_url)+ ):+ return protocol++ if isinstance(store_protocol, tuple):+ for store_protocol_item in store_protocol:+ if normalized := normalize_protocol(store_protocol_item):+ return normalized+ return "-".join(store_protocol)++ return normalize_protocol(store_protocol) or store_protocol@propertydef root_path(self) -> str | None:@@ -335,18 +358,50 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):return isinstance(var, AbstractFileSystem)-def normalize_protocol(protocol: str) -> KNOWN_STORAGE_TYPES | str:- """Normalize the protocol to a known storage type."""- protocol = protocol.strip().lower()- if "s3" in protocol:- return "s3"- elif "gcs" in protocol or "gs" in protocol:- return "gcs"- elif "azure" in protocol:- return "azure"- elif "http" in protocol:- return "http"- elif "file" in protocol:- return "file"- else:- return protocol+_PROTOCOL_MAP: dict[str, KNOWN_STORAGE_TYPES] = {+ "s3": "s3",+ "s3a": "s3",+ "gs": "gcs",+ "gcs": "gcs",+ "abfs": "azure",+ "abfss": "azure",+ "az": "azure",+ "adl": "azure",+ "http": "http",+ "https": "http",+ "file": "file",+ "local": "file",+ "memory": "in-memory",+ "r2": "cloudflare",+}++# Specific provider patterns checked before generic ones (e.g. S3),+# since S3-compatible services may also have "s3" in their URL.+# The order of the patterns is important, the first pattern that matches will be used.+_URL_PATTERNS: list[tuple[str, CLOUD_STORAGE_TYPES]] = [+ ("cloudflare", "cloudflare"),+ ("r2.", "cloudflare"),+ ("cwobject", "coreweave"),+ ("cwlota", "coreweave"),+ ("coreweave", "coreweave"),+ ("blob.core.windows", "azure"),+ ("azure", "azure"),+ ("googleapis", "gcs"),+ ("storage.google", "gcs"),+ ("s3", "s3"),+ ("amazonaws", "s3"),+]+++def detect_protocol_from_url(url: str) -> CLOUD_STORAGE_TYPES | None:+ """Detect the storage provider from an endpoint URL."""+ url = url.strip().lower()+ for pattern, protocol in _URL_PATTERNS:+ if pattern in url:+ return protocol+ return None+++def normalize_protocol(protocol: str) -> KNOWN_STORAGE_TYPES | None:+ """Normalize a protocol string (e.g. 's3a', 'gs', 'abfs') to a known storage type."""+ return _PROTOCOL_MAP.get(protocol.strip().lower())diff --git a/tests/_data/_external_storage/test_get_storage.py b/tests/_data/_external_storage/test_get_storage.pyindex 350a2e9735..1bf597b402 100644--- a/tests/_data/_external_storage/test_get_storage.py+++ b/tests/_data/_external_storage/test_get_storage.py@@ -152,8 +152,8 @@ class TestStorageBackendToStorageNamespace:assert result == snapshot(StorageNamespace(name=VariableName("mem_fs"),- display_name="Memory",- protocol="memory",+ display_name="In-memory",+ protocol="in-memory",root_path="/",storage_entries=[],)diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.pyindex e3f79726d8..09c3971c02 100644--- a/tests/_data/_external_storage/test_storage_models.py+++ b/tests/_data/_external_storage/test_storage_models.py@@ -14,6 +14,7 @@ from marimo._data._external_storage.models import DownloadResult, StorageEntryfrom marimo._data._external_storage.storage import (FsspecFilesystem,Obstore,+ detect_protocol_from_url,normalize_protocol,)from marimo._dependencies.dependencies import DependencyManager@@ -1024,8 +1025,7 @@ class TestFsspecFilesystemIntegration:fs = MemoryFileSystem()backend = FsspecFilesystem(fs, VariableName("mem_fs"))- # MemoryFileSystem protocol is "memory", which doesn't match known types- assert backend.protocol == "memory"+ assert backend.protocol == "in-memory"@pytest.mark.skipif(not HAS_OBSTORE, reason="obstore not installed")@@ -1152,28 +1152,51 @@ class TestNormalizeProtocol:("s3", "s3"),("s3a", "s3"),("S3", "s3"),+ ("gs", "gcs"),("gcs", "gcs"),- ("GCS", "gcs"),- ("gs-gcs", "gcs"),- ("azure", "azure"),- ("Azure", "azure"),- ("abfs-azure", "azure"),+ ("abfs", "azure"),+ ("abfss", "azure"),+ ("az", "azure"),("http", "http"),- ("HTTP", "http"),- ("https-http", "http"),+ ("https", "http"),("file", "file"),- ("FILE", "file"),- ("local-file", "file"),- ("ftp", "ftp"),- ("custom", "custom"),+ ("local", "file"),+ ("memory", "in-memory"),+ ("r2", "cloudflare"),(" s3 ", "s3"),- (" gcs ", "gcs"),+ ("unknown", None),+ ("ftp", None),],)- def test_normalize_protocol(self, protocol: str, expected: str) -> None:+ def test_normalize_protocol(+ self, protocol: str, expected: str | None+ ) -> None:assert normalize_protocol(protocol) == expected+class TestDetectProtocolFromUrl:+ @pytest.mark.parametrize(+ ("url", "expected"),+ [+ ("https://account.r2.cloudflarestorage.com", "cloudflare"),+ ("https://s3.amazonaws.com", "s3"),+ ("https://s3.us-east-1.amazonaws.com", "s3"),+ ("https://storage.googleapis.com", "gcs"),+ ("https://account.blob.core.windows.net", "azure"),+ ("https://minio.example.com", None),+ ("https://my-custom-endpoint.com", None),+ (+ "https://s3.cloudflare.com",+ "cloudflare",+ ), # Although there is S3, it will match the cloudflare pattern first+ ],+ )+ def test_detect_protocol_from_url(+ self, url: str, expected: str | None+ ) -> None:+ assert detect_protocol_from_url(url) == expected++# --- Helpers ---