A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix cached self entries fsspec (#8863)
Shahmir Varqha3cf9f4aff789
diff --git a/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap b/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snapindex bb47f0d5ea..127ad01812 100644--- a/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap+++ b/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap@@ -85,7 +85,7 @@ store = GCSStore("my-bucket")"exports[`generateStorageCode > Google Drive > with browser auth (no credentials) 1`] = `"from gdrive_fsspec import GoogleDriveFileSystem-fs = GoogleDriveFileSystem(token="browser")"+fs = GoogleDriveFileSystem(token="browser", use_listings_cache=False)"`;exports[`generateStorageCode > Google Drive > with service account credentials 1`] = `@@ -93,7 +93,7 @@ exports[`generateStorageCode > Google Drive > with service account credentials 1import json_creds = json.loads("""{"type": "service_account", "client_email": "[email protected]"}""")-fs = GoogleDriveFileSystem(creds=_creds, token="service_account")"+fs = GoogleDriveFileSystem(creds=_creds, token="service_account", use_listings_cache=False)"`;exports[`generateStorageCode > S3 > basic connection with all fields 1`] = `diff --git a/frontend/src/components/editor/connections/storage/as-code.ts b/frontend/src/components/editor/connections/storage/as-code.tsindex 7b8b8fed5c..49e5d7c41e 100644--- a/frontend/src/components/editor/connections/storage/as-code.ts+++ b/frontend/src/components/editor/connections/storage/as-code.ts@@ -173,13 +173,13 @@ function generateGDriveCode();const code = dedent(`_creds = json.loads("""${connection.credentials_json?.startsWith("ENV:") ? `{${creds}}` : connection.credentials_json}""")- fs = GoogleDriveFileSystem(creds=_creds, token="service_account")+ fs = GoogleDriveFileSystem(creds=_creds, token="service_account", use_listings_cache=False)`);return { imports, code };}const code = dedent(`- fs = GoogleDriveFileSystem(token="browser")+ fs = GoogleDriveFileSystem(token="browser", use_listings_cache=False)`);return { imports, code };}diff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.pyindex 1fb0f94195..65199dbc6d 100644--- a/marimo/_data/_external_storage/storage.py+++ b/marimo/_data/_external_storage/storage.py@@ -253,9 +253,16 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):if prefix is None:prefix = ""- files = self.store.ls(path=prefix, detail=True)- if not isinstance(files, list):- raise ValueError(f"Files is not a list: {files}")+ files = self._list_files(prefix)+ normalized_prefix = self._normalize_path(prefix)+ if self._has_self_entry(files, normalized_prefix):+ LOGGER.debug(+ "Detected self-entry for prefix %s, invalidating cache and retrying",+ prefix,+ )+ self._invalidate_listing_cache_for_prefix(prefix)+ files = self._list_files(prefix)+total_files = len(files)if total_files > limit:LOGGER.debug(@@ -273,6 +280,63 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):return storage_entries+ def _normalize_path(self, path: str) -> str:+ # Match fsspec path handling: strip protocol and trailing slashes,+ # but preserve leading slash for absolute paths (e.g. "/tmp").+ path = path.strip()+ strip_protocol = getattr(self.store, "_strip_protocol", None)+ if callable(strip_protocol):+ stripped = strip_protocol(path)+ if isinstance(stripped, str):+ path = stripped+ return path.rstrip("/")++ def _list_files(self, prefix: str) -> list[Any]:+ files = self.store.ls(path=prefix, detail=True)+ if not isinstance(files, list):+ raise ValueError(f"Files is not a list: {files}")+ return files++ def _has_self_entry(+ self, files: list[Any], normalized_prefix: str+ ) -> bool:+ # Stale dircache listings tend to echo only the queried path (length 1).+ # Avoid scanning large result sets for an exact prefix match.+ if not normalized_prefix or len(files) != 1:+ return False+ return self._is_self_entry(files[0], normalized_prefix)++ def _is_self_entry(self, file: Any, normalized_prefix: str) -> bool:+ if not isinstance(file, dict):+ return False+ name = file.get("name")+ if not isinstance(name, str):+ return False+ return self._normalize_path(name) == normalized_prefix++ def _invalidate_listing_cache_for_prefix(self, prefix: str) -> None:+ dircache = getattr(self.store, "dircache", None)+ if dircache is None:+ return++ # Clear the minimum cache surface (target, parent, and root) that can+ # contribute stale self-entries before retrying list().+ cache_keys: set[str] = {self._normalize_path(prefix), ""}+ parent_fn = getattr(self.store, "_parent", None)+ if callable(parent_fn):+ parent = parent_fn(prefix)+ if isinstance(parent, str):+ cache_keys.add(self._normalize_path(parent))++ for key in cache_keys:+ try:+ dircache.pop(key, None)+ except Exception:+ # Some custom mappings may not support pop semantics.+ LOGGER.debug(+ "Failed to clear fsspec cache key %s for listing", key+ )+def _identify_kind(self, entry_type: str) -> Literal["file", "directory"]:entry_type = entry_type.strip().lower()if entry_type == "file":diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.pyindex f6be34f1e4..e1e0efc4ac 100644--- a/tests/_data/_external_storage/test_storage_models.py+++ b/tests/_data/_external_storage/test_storage_models.py@@ -616,6 +616,109 @@ class TestFsspecFilesystem:mock_store.ls.assert_called_once_with(path="", detail=True)+ def test_list_entries_retries_when_self_entry_detected(self) -> None:+ mock_store = MagicMock()+ mock_store.protocol = "file"+ mock_store.dircache = {"": [{"name": "folder"}], "folder": []}+ mock_store._parent = lambda _path: ""+ mock_store.ls.side_effect = [+ [+ {+ "name": "folder",+ "size": 0,+ "type": "directory",+ "mtime": None,+ }+ ],+ [+ {+ "name": "folder/file.txt",+ "size": 100,+ "type": "file",+ "mtime": 1234567890.0,+ }+ ],+ ]++ backend = self._make_backend(mock_store)+ result = backend.list_entries(prefix="folder")++ assert mock_store.ls.call_count == 2+ assert "" not in mock_store.dircache+ assert "folder" not in mock_store.dircache+ assert result == snapshot(+ [+ StorageEntry(+ path="folder/file.txt",+ kind="file",+ size=100,+ last_modified=1234567890.0,+ metadata={},+ mime_type="text/plain",+ ),+ ]+ )++ def test_list_entries_returns_multi_file_list_without_self_scan(+ self,+ ) -> None:+ """Multi-entry listings skip self-entry handling (O(1) path); all rows pass through."""+ mock_store = MagicMock()+ mock_store.protocol = "file"+ mock_store.ls.return_value = [+ {+ "name": "folder",+ "size": 0,+ "type": "directory",+ "mtime": None,+ },+ {+ "name": "folder/folder",+ "size": 0,+ "type": "directory",+ "mtime": None,+ },+ {+ "name": "folder/file.txt",+ "size": 1,+ "type": "file",+ "mtime": None,+ },+ ]++ backend = self._make_backend(mock_store)+ result = backend.list_entries(prefix="folder")++ mock_store.ls.assert_called_once_with(path="folder", detail=True)+ assert result == snapshot(+ [+ StorageEntry(+ path="folder",+ kind="directory",+ size=0,+ last_modified=None,+ metadata={},+ mime_type=None,+ ),+ StorageEntry(+ path="folder/folder",+ kind="directory",+ size=0,+ last_modified=None,+ metadata={},+ mime_type=None,+ ),+ StorageEntry(+ path="folder/file.txt",+ kind="file",+ size=1,+ last_modified=None,+ metadata={},+ mime_type="text/plain",+ ),+ ]+ )+def test_list_entries_respects_limit(self) -> None:mock_store = MagicMock()files = [