Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

UI fix for SFTP (and a fsspec plugin for ssh) (#10977)

vincent d warmerdam5713d7373361
diff --git a/docs/guides/working_with_data/remote_storage.md b/docs/guides/working_with_data/remote_storage.md
index 62e314f4bd..153ae6ef1f 100644
--- a/docs/guides/working_with_data/remote_storage.md
+++ b/docs/guides/working_with_data/remote_storage.md
@@ -15,7 +15,7 @@ marimo auto-discovers variables that are instances of:
| Library | Base class | Example stores |
|---------|-----------|----------------|
| [obstore](https://developmentseed.org/obstore/) | `obstore.store.ObjectStore` | `S3Store`, `GCSStore`, `AzureStore`, `HTTPStore`, `LocalStore`, `MemoryStore` |
-| [fsspec](https://filesystem-spec.readthedocs.io/) | `fsspec.AbstractFileSystem` | `S3FileSystem`, `GithubFileSystem`, `FTPFileSystem`, `DatabricksFileSystem`, and [many more](https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations) |
+| [fsspec](https://filesystem-spec.readthedocs.io/) | `fsspec.AbstractFileSystem` | `S3FileSystem`, `GithubFileSystem`, `FTPFileSystem`, `SFTPFileSystem`, `DatabricksFileSystem`, and [many more](https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations) |
| [huggingface_hub](https://huggingface.co/docs/huggingface_hub) | `huggingface_hub.HfApi` | Browse the Hugging Face Hub (datasets, models, spaces, buckets) |
@@ -89,6 +89,47 @@ from fsspec.implementations.github import GithubFileSystem
repo = GithubFileSystem(org="marimo-team", repo="marimo")
```
+#### SSH / SFTP
+
+Browse files on a remote machine over SSH with fsspec's `SFTPFileSystem`, which requires [paramiko](https://www.paramiko.org/) (`pip install paramiko`):
+
+```python
+from fsspec.implementations.sftp import SFTPFileSystem
+
+server = SFTPFileSystem("my-server.example.com", username="me")
+```
+
+By default, this uses your SSH agent and the default keys in `~/.ssh`. To use a specific key, pass `key_filename`. Note that paramiko doesn't expand `~` or read `~/.ssh/config`, so use the full key path and the real hostname:
+
+```python
+import os
+
+from fsspec.implementations.sftp import SFTPFileSystem
+
+server = SFTPFileSystem(
+ "my-server.example.com",
+ username="me",
+ key_filename="/home/<user>>/.ssh/<filename>",
+ passphrase=os.environ.get("SSH_KEY_PASSPHRASE"), # only for encrypted keys
+)
+```
+
+Other keyword arguments are passed to [`paramiko.SSHClient.connect`](https://docs.paramiko.org/en/stable/api/client.html#paramiko.client.SSHClient.connect), e.g. `port` or `password`.
+
+The panel opens in your home directory on the server. To start in a different folder, wrap the filesystem in fsspec's `DirFileSystem`; paths you use in code are then relative to that folder too:
+
+```python
+from fsspec.implementations.dirfs import DirFileSystem
+from fsspec.implementations.sftp import SFTPFileSystem
+
+project = DirFileSystem(
+ path="/data/project",
+ fs=SFTPFileSystem("my-server.example.com", username="me"),
+)
+```
+
+Note that `DirFileSystem` works with any fsspec filesystem, not just SFTP.
+
#### Hugging Face Hub
```python
diff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.py
index bbe706d9bf..b6858382b2 100644
--- a/marimo/_data/_external_storage/storage.py
+++ b/marimo/_data/_external_storage/storage.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import asyncio
import re
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
from functools import cached_property
from typing import TYPE_CHECKING, Any, Literal, cast
@@ -308,7 +308,14 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):
return path.rstrip("/")
def _list_files(self, prefix: str) -> list[Any]:
- files = self.store.ls(path=prefix, detail=True)
+ try:
+ files = self.store.ls(path=prefix, detail=True)
+ except FileNotFoundError:
+ # SFTP filesystems (e.g. sshfs) have an empty root marker but
+ # can't list "", so fall back to the working (home) directory
+ if prefix:
+ raise
+ files = self.store.ls(path=".", detail=True)
if not isinstance(files, list):
raise ValueError(f"Files is not a list: {files}")
return files
@@ -396,11 +403,20 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):
else:
resolved_kind = self._identify_kind(entry_type)
- resolved_path = name or ""
+ # Some filesystems (e.g. sshfs) return mtime as a datetime;
+ # fsspec times are UTC, so treat naive datetimes as UTC
+ mtime = file.get("mtime")
+ if isinstance(mtime, datetime):
+ mtime = mtime.replace(
+ tzinfo=mtime.tzinfo or timezone.utc
+ ).timestamp()
+
+ # Entries listed from the "." fallback are prefixed with "./"
+ resolved_path = (name or "").removeprefix("./")
return StorageEntry(
path=resolved_path,
size=size or 0,
- last_modified=file.get("mtime"),
+ last_modified=mtime,
kind=resolved_kind,
metadata=entry_meta,
mime_type=guess_mime_type(resolved_path)
diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.py
index 0d84fef051..e2ac67b462 100644
--- a/tests/_data/_external_storage/test_storage_models.py
+++ b/tests/_data/_external_storage/test_storage_models.py
@@ -745,6 +745,32 @@ class TestFsspecFilesystem:
mock_store.ls.assert_called_once_with(path="", detail=True)
+ def test_list_entries_root_falls_back_to_cwd(self) -> None:
+ # SFTP filesystems can't list "" but can list "."
+ mock_store = MagicMock()
+ mock_store.ls.side_effect = [
+ FileNotFoundError(),
+ [{"name": "./a.txt", "size": 1, "type": "file"}],
+ ]
+
+ backend = self._make_backend(mock_store)
+ result = backend.list_entries(prefix=None)
+
+ assert mock_store.ls.call_args_list == [
+ ((), {"path": "", "detail": True}),
+ ((), {"path": ".", "detail": True}),
+ ]
+ assert [e.path for e in result.entries] == ["a.txt"]
+
+ def test_list_entries_not_found_below_root_raises(self) -> None:
+ mock_store = MagicMock()
+ mock_store.ls.side_effect = FileNotFoundError()
+
+ backend = self._make_backend(mock_store)
+ with pytest.raises(FileNotFoundError):
+ backend.list_entries(prefix="missing")
+ mock_store.ls.assert_called_once_with(path="missing", detail=True)
+
def test_list_entries_retries_when_self_entry_detected(self) -> None:
mock_store = MagicMock()
mock_store.protocol = "file"
@@ -1112,6 +1138,24 @@ class TestFsspecFilesystem:
)
)
+ @pytest.mark.parametrize(
+ "mtime",
+ [
+ datetime(2024, 1, 1, tzinfo=timezone.utc),
+ datetime(2024, 1, 1), # naive datetimes are treated as UTC
+ ],
+ )
+ def test_create_storage_entry_datetime_mtime(
+ self, mtime: datetime
+ ) -> None:
+ # e.g. sshfs returns mtime as a datetime rather than a float
+ backend = self._make_backend(MagicMock())
+
+ entry = backend._create_storage_entry(
+ {"name": "a.txt", "size": 1, "type": "file", "mtime": mtime}
+ )
+ assert entry.last_modified == 1704067200.0
+
async def test_get_entry(self) -> None:
mock_store = MagicMock()
mock_store.info.return_value = {