Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix cached self entries fsspec (#8863)

Shahmir Varqha3cf9f4aff789
diff --git a/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap b/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap
index bb47f0d5ea..127ad01812 100644
--- a/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap
+++ b/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap
@@ -85,7 +85,7 @@ store = GCSStore("my-bucket")"
exports[`generateStorageCode > Google Drive > with browser auth (no credentials) 1`] = `
"from gdrive_fsspec import GoogleDriveFileSystem
-fs = GoogleDriveFileSystem(token="browser")"
+fs = GoogleDriveFileSystem(token="browser", use_listings_cache=False)"
`;
exports[`generateStorageCode > Google Drive > with service account credentials 1`] = `
@@ -93,7 +93,7 @@ exports[`generateStorageCode > Google Drive > with service account credentials 1
import json
_creds = json.loads("""{"type": "service_account", "client_email": "[email protected]"}""")
-fs = GoogleDriveFileSystem(creds=_creds, token="service_account")"
+fs = GoogleDriveFileSystem(creds=_creds, token="service_account", use_listings_cache=False)"
`;
exports[`generateStorageCode > S3 > basic connection with all fields 1`] = `
diff --git a/frontend/src/components/editor/connections/storage/as-code.ts b/frontend/src/components/editor/connections/storage/as-code.ts
index 7b8b8fed5c..49e5d7c41e 100644
--- a/frontend/src/components/editor/connections/storage/as-code.ts
+++ b/frontend/src/components/editor/connections/storage/as-code.ts
@@ -173,13 +173,13 @@ function generateGDriveCode(
);
const code = dedent(`
_creds = json.loads("""${connection.credentials_json?.startsWith("ENV:") ? `{${creds}}` : connection.credentials_json}""")
- fs = GoogleDriveFileSystem(creds=_creds, token="service_account")
+ fs = GoogleDriveFileSystem(creds=_creds, token="service_account", use_listings_cache=False)
`);
return { imports, code };
}
const code = dedent(`
- fs = GoogleDriveFileSystem(token="browser")
+ fs = GoogleDriveFileSystem(token="browser", use_listings_cache=False)
`);
return { imports, code };
}
diff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.py
index 1fb0f94195..65199dbc6d 100644
--- a/marimo/_data/_external_storage/storage.py
+++ b/marimo/_data/_external_storage/storage.py
@@ -253,9 +253,16 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):
if prefix is None:
prefix = ""
- files = self.store.ls(path=prefix, detail=True)
- if not isinstance(files, list):
- raise ValueError(f"Files is not a list: {files}")
+ files = self._list_files(prefix)
+ normalized_prefix = self._normalize_path(prefix)
+ if self._has_self_entry(files, normalized_prefix):
+ LOGGER.debug(
+ "Detected self-entry for prefix %s, invalidating cache and retrying",
+ prefix,
+ )
+ self._invalidate_listing_cache_for_prefix(prefix)
+ files = self._list_files(prefix)
+
total_files = len(files)
if total_files > limit:
LOGGER.debug(
@@ -273,6 +280,63 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):
return storage_entries
+ def _normalize_path(self, path: str) -> str:
+ # Match fsspec path handling: strip protocol and trailing slashes,
+ # but preserve leading slash for absolute paths (e.g. "/tmp").
+ path = path.strip()
+ strip_protocol = getattr(self.store, "_strip_protocol", None)
+ if callable(strip_protocol):
+ stripped = strip_protocol(path)
+ if isinstance(stripped, str):
+ path = stripped
+ return path.rstrip("/")
+
+ def _list_files(self, prefix: str) -> list[Any]:
+ files = self.store.ls(path=prefix, detail=True)
+ if not isinstance(files, list):
+ raise ValueError(f"Files is not a list: {files}")
+ return files
+
+ def _has_self_entry(
+ self, files: list[Any], normalized_prefix: str
+ ) -> bool:
+ # Stale dircache listings tend to echo only the queried path (length 1).
+ # Avoid scanning large result sets for an exact prefix match.
+ if not normalized_prefix or len(files) != 1:
+ return False
+ return self._is_self_entry(files[0], normalized_prefix)
+
+ def _is_self_entry(self, file: Any, normalized_prefix: str) -> bool:
+ if not isinstance(file, dict):
+ return False
+ name = file.get("name")
+ if not isinstance(name, str):
+ return False
+ return self._normalize_path(name) == normalized_prefix
+
+ def _invalidate_listing_cache_for_prefix(self, prefix: str) -> None:
+ dircache = getattr(self.store, "dircache", None)
+ if dircache is None:
+ return
+
+ # Clear the minimum cache surface (target, parent, and root) that can
+ # contribute stale self-entries before retrying list().
+ cache_keys: set[str] = {self._normalize_path(prefix), ""}
+ parent_fn = getattr(self.store, "_parent", None)
+ if callable(parent_fn):
+ parent = parent_fn(prefix)
+ if isinstance(parent, str):
+ cache_keys.add(self._normalize_path(parent))
+
+ for key in cache_keys:
+ try:
+ dircache.pop(key, None)
+ except Exception:
+ # Some custom mappings may not support pop semantics.
+ LOGGER.debug(
+ "Failed to clear fsspec cache key %s for listing", key
+ )
+
def _identify_kind(self, entry_type: str) -> Literal["file", "directory"]:
entry_type = entry_type.strip().lower()
if entry_type == "file":
diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.py
index f6be34f1e4..e1e0efc4ac 100644
--- a/tests/_data/_external_storage/test_storage_models.py
+++ b/tests/_data/_external_storage/test_storage_models.py
@@ -616,6 +616,109 @@ class TestFsspecFilesystem:
mock_store.ls.assert_called_once_with(path="", detail=True)
+ def test_list_entries_retries_when_self_entry_detected(self) -> None:
+ mock_store = MagicMock()
+ mock_store.protocol = "file"
+ mock_store.dircache = {"": [{"name": "folder"}], "folder": []}
+ mock_store._parent = lambda _path: ""
+ mock_store.ls.side_effect = [
+ [
+ {
+ "name": "folder",
+ "size": 0,
+ "type": "directory",
+ "mtime": None,
+ }
+ ],
+ [
+ {
+ "name": "folder/file.txt",
+ "size": 100,
+ "type": "file",
+ "mtime": 1234567890.0,
+ }
+ ],
+ ]
+
+ backend = self._make_backend(mock_store)
+ result = backend.list_entries(prefix="folder")
+
+ assert mock_store.ls.call_count == 2
+ assert "" not in mock_store.dircache
+ assert "folder" not in mock_store.dircache
+ assert result == snapshot(
+ [
+ StorageEntry(
+ path="folder/file.txt",
+ kind="file",
+ size=100,
+ last_modified=1234567890.0,
+ metadata={},
+ mime_type="text/plain",
+ ),
+ ]
+ )
+
+ def test_list_entries_returns_multi_file_list_without_self_scan(
+ self,
+ ) -> None:
+ """Multi-entry listings skip self-entry handling (O(1) path); all rows pass through."""
+ mock_store = MagicMock()
+ mock_store.protocol = "file"
+ mock_store.ls.return_value = [
+ {
+ "name": "folder",
+ "size": 0,
+ "type": "directory",
+ "mtime": None,
+ },
+ {
+ "name": "folder/folder",
+ "size": 0,
+ "type": "directory",
+ "mtime": None,
+ },
+ {
+ "name": "folder/file.txt",
+ "size": 1,
+ "type": "file",
+ "mtime": None,
+ },
+ ]
+
+ backend = self._make_backend(mock_store)
+ result = backend.list_entries(prefix="folder")
+
+ mock_store.ls.assert_called_once_with(path="folder", detail=True)
+ assert result == snapshot(
+ [
+ StorageEntry(
+ path="folder",
+ kind="directory",
+ size=0,
+ last_modified=None,
+ metadata={},
+ mime_type=None,
+ ),
+ StorageEntry(
+ path="folder/folder",
+ kind="directory",
+ size=0,
+ last_modified=None,
+ metadata={},
+ mime_type=None,
+ ),
+ StorageEntry(
+ path="folder/file.txt",
+ kind="file",
+ size=1,
+ last_modified=None,
+ metadata={},
+ mime_type="text/plain",
+ ),
+ ]
+ )
+
def test_list_entries_respects_limit(self) -> None:
mock_store = MagicMock()
files = [