A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
UI fix for SFTP (and a fsspec plugin for ssh) (#10977)
vincent d warmerdam5713d7373361
diff --git a/docs/guides/working_with_data/remote_storage.md b/docs/guides/working_with_data/remote_storage.mdindex 62e314f4bd..153ae6ef1f 100644--- a/docs/guides/working_with_data/remote_storage.md+++ b/docs/guides/working_with_data/remote_storage.md@@ -15,7 +15,7 @@ marimo auto-discovers variables that are instances of:| Library | Base class | Example stores ||---------|-----------|----------------|| [obstore](https://developmentseed.org/obstore/) | `obstore.store.ObjectStore` | `S3Store`, `GCSStore`, `AzureStore`, `HTTPStore`, `LocalStore`, `MemoryStore` |-| [fsspec](https://filesystem-spec.readthedocs.io/) | `fsspec.AbstractFileSystem` | `S3FileSystem`, `GithubFileSystem`, `FTPFileSystem`, `DatabricksFileSystem`, and [many more](https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations) |+| [fsspec](https://filesystem-spec.readthedocs.io/) | `fsspec.AbstractFileSystem` | `S3FileSystem`, `GithubFileSystem`, `FTPFileSystem`, `SFTPFileSystem`, `DatabricksFileSystem`, and [many more](https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations) || [huggingface_hub](https://huggingface.co/docs/huggingface_hub) | `huggingface_hub.HfApi` | Browse the Hugging Face Hub (datasets, models, spaces, buckets) |@@ -89,6 +89,47 @@ from fsspec.implementations.github import GithubFileSystemrepo = GithubFileSystem(org="marimo-team", repo="marimo")```+#### SSH / SFTP++Browse files on a remote machine over SSH with fsspec's `SFTPFileSystem`, which requires [paramiko](https://www.paramiko.org/) (`pip install paramiko`):++```python+from fsspec.implementations.sftp import SFTPFileSystem++server = SFTPFileSystem("my-server.example.com", username="me")+```++By default, this uses your SSH agent and the default keys in `~/.ssh`. To use a specific key, pass `key_filename`. Note that paramiko doesn't expand `~` or read `~/.ssh/config`, so use the full key path and the real hostname:++```python+import os++from fsspec.implementations.sftp import SFTPFileSystem++server = SFTPFileSystem(+ "my-server.example.com",+ username="me",+ key_filename="/home/<user>>/.ssh/<filename>",+ passphrase=os.environ.get("SSH_KEY_PASSPHRASE"), # only for encrypted keys+)+```++Other keyword arguments are passed to [`paramiko.SSHClient.connect`](https://docs.paramiko.org/en/stable/api/client.html#paramiko.client.SSHClient.connect), e.g. `port` or `password`.++The panel opens in your home directory on the server. To start in a different folder, wrap the filesystem in fsspec's `DirFileSystem`; paths you use in code are then relative to that folder too:++```python+from fsspec.implementations.dirfs import DirFileSystem+from fsspec.implementations.sftp import SFTPFileSystem++project = DirFileSystem(+ path="/data/project",+ fs=SFTPFileSystem("my-server.example.com", username="me"),+)+```++Note that `DirFileSystem` works with any fsspec filesystem, not just SFTP.+#### Hugging Face Hub```pythondiff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.pyindex bbe706d9bf..b6858382b2 100644--- a/marimo/_data/_external_storage/storage.py+++ b/marimo/_data/_external_storage/storage.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport asyncioimport re-from datetime import timedelta+from datetime import datetime, timedelta, timezonefrom functools import cached_propertyfrom typing import TYPE_CHECKING, Any, Literal, cast@@ -308,7 +308,14 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):return path.rstrip("/")def _list_files(self, prefix: str) -> list[Any]:- files = self.store.ls(path=prefix, detail=True)+ try:+ files = self.store.ls(path=prefix, detail=True)+ except FileNotFoundError:+ # SFTP filesystems (e.g. sshfs) have an empty root marker but+ # can't list "", so fall back to the working (home) directory+ if prefix:+ raise+ files = self.store.ls(path=".", detail=True)if not isinstance(files, list):raise ValueError(f"Files is not a list: {files}")return files@@ -396,11 +403,20 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):else:resolved_kind = self._identify_kind(entry_type)- resolved_path = name or ""+ # Some filesystems (e.g. sshfs) return mtime as a datetime;+ # fsspec times are UTC, so treat naive datetimes as UTC+ mtime = file.get("mtime")+ if isinstance(mtime, datetime):+ mtime = mtime.replace(+ tzinfo=mtime.tzinfo or timezone.utc+ ).timestamp()++ # Entries listed from the "." fallback are prefixed with "./"+ resolved_path = (name or "").removeprefix("./")return StorageEntry(path=resolved_path,size=size or 0,- last_modified=file.get("mtime"),+ last_modified=mtime,kind=resolved_kind,metadata=entry_meta,mime_type=guess_mime_type(resolved_path)diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.pyindex 0d84fef051..e2ac67b462 100644--- a/tests/_data/_external_storage/test_storage_models.py+++ b/tests/_data/_external_storage/test_storage_models.py@@ -745,6 +745,32 @@ class TestFsspecFilesystem:mock_store.ls.assert_called_once_with(path="", detail=True)+ def test_list_entries_root_falls_back_to_cwd(self) -> None:+ # SFTP filesystems can't list "" but can list "."+ mock_store = MagicMock()+ mock_store.ls.side_effect = [+ FileNotFoundError(),+ [{"name": "./a.txt", "size": 1, "type": "file"}],+ ]++ backend = self._make_backend(mock_store)+ result = backend.list_entries(prefix=None)++ assert mock_store.ls.call_args_list == [+ ((), {"path": "", "detail": True}),+ ((), {"path": ".", "detail": True}),+ ]+ assert [e.path for e in result.entries] == ["a.txt"]++ def test_list_entries_not_found_below_root_raises(self) -> None:+ mock_store = MagicMock()+ mock_store.ls.side_effect = FileNotFoundError()++ backend = self._make_backend(mock_store)+ with pytest.raises(FileNotFoundError):+ backend.list_entries(prefix="missing")+ mock_store.ls.assert_called_once_with(path="missing", detail=True)+def test_list_entries_retries_when_self_entry_detected(self) -> None:mock_store = MagicMock()mock_store.protocol = "file"@@ -1112,6 +1138,24 @@ class TestFsspecFilesystem:))+ @pytest.mark.parametrize(+ "mtime",+ [+ datetime(2024, 1, 1, tzinfo=timezone.utc),+ datetime(2024, 1, 1), # naive datetimes are treated as UTC+ ],+ )+ def test_create_storage_entry_datetime_mtime(+ self, mtime: datetime+ ) -> None:+ # e.g. sshfs returns mtime as a datetime rather than a float+ backend = self._make_backend(MagicMock())++ entry = backend._create_storage_entry(+ {"name": "a.txt", "size": 1, "type": "file", "mtime": mtime}+ )+ assert entry.last_modified == 1704067200.0+async def test_get_entry(self) -> None:mock_store = MagicMock()mock_store.info.return_value = {