Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
adding support for SFTP (#474)
Nahuel Nuceraed68ee924cb1
diff --git a/Dockerfile b/Dockerfileindex 415b35e7..5cb8de0d 100644--- a/Dockerfile+++ b/Dockerfile@@ -8,6 +8,7 @@ RUN pip install --no-cache-dir --no-compile \"pyyaml>=6.0" \"cryptography>=41.0" \"pymysql>=1.1" \+ "asyncssh>=2.14" \"boto3>=1.34" \"awscli"@@ -61,7 +62,7 @@ ENV GATEWAY_PORT=4566 \LAMBDA_EXECUTOR=local \PYTHONUNBUFFERED=1-EXPOSE 4566+EXPOSE 4566 2222# Pure Python healthcheck — no curl dependencyHEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \diff --git a/ministack/app.py b/ministack/app.pyindex f692c886..dc54740c 100644--- a/ministack/app.py+++ b/ministack/app.py@@ -537,9 +537,37 @@ async def _handle_pre_body_request(method: str, path: str, headers: dict, query_if response is not None:return response+ response = _handle_transfer_sftp_ports_request(method, path)+ if response is not None:+ return response+return await _handle_admin_reset(path, method, query_params)+def _handle_transfer_sftp_ports_request(method: str, path: str):+ """`GET /_ministack/transfer/sftp-ports` returns ``{shared, per_server}``.++ boto3's DescribeServer drops fields not in the AWS spec, so this+ admin endpoint is how tests (and humans) discover which ports+ MiniStack's SFTP listeners ended up on — particularly relevant+ when ``SFTP_PORT_PER_SERVER=1`` allocates ports dynamically from+ ``SFTP_BASE_PORT``.+ """+ if path != "/_ministack/transfer/sftp-ports" or method != "GET":+ return None+ try:+ from ministack.services import transfer+ body = {+ "enabled": transfer._sftp_enabled(),+ "port_per_server": transfer._port_per_server(),+ "shared_port": transfer._shared_port() if transfer._sftp_enabled() else None,+ "per_server": dict(transfer._sftp_per_server_ports),+ }+ except Exception as e:+ return 500, {"Content-Type": "application/json"}, json.dumps({"message": str(e)}).encode()+ return 200, {"Content-Type": "application/json"}, json.dumps(body).encode()++# ---------------------------------------------------------------------------# Tier 2 — Post-body shortcuts (body required, before generic routing)# ---------------------------------------------------------------------------@@ -1085,6 +1113,15 @@ async def _handle_lifespan(scope, receive, send):_run_init_scripts()if PERSIST_STATE:_load_persisted_state()+ # Start the Transfer Family SFTP listener after persistence is+ # loaded (so any restored Transfer servers/users are visible to+ # the SSH auth callback). Cheap no-op if asyncssh isn't+ # installed or SFTP_ENABLED=0.+ try:+ from ministack.services import transfer+ await transfer.sftp_start()+ except Exception as e:+ logger.warning("Transfer SFTP startup failed: %s", e)await send({"type": "lifespan.startup.complete"})logger.info("Ready.")for svc in SERVICE_HANDLERS:@@ -1119,6 +1156,11 @@ async def _handle_lifespan(scope, receive, send):if mod_name in _loaded_modules:save_dict[key] = _loaded_modules[mod_name].get_statesave_all(save_dict)+ try:+ from ministack.services import transfer+ await transfer.sftp_stop()+ except Exception as e:+ logger.debug("Transfer SFTP shutdown error: %s", e)_stop_docker_containers()await send({"type": "lifespan.shutdown.complete"})returndiff --git a/ministack/services/apigateway.py b/ministack/services/apigateway.pyindex e68f356c..794c6117 100644--- a/ministack/services/apigateway.py+++ b/ministack/services/apigateway.py@@ -1200,8 +1200,17 @@ async def _invoke_ws_lambda(api_id: str, account_id: str, route: dict, stage: struntime = func_config.get("Runtime", "")code_zip = func_data.get("code_zip")if code_zip and runtime.startswith(("python", "nodejs")):- worker_key = f"{func_name}:{qualifier}" if qualifier else func_name- worker = get_or_create_worker(worker_key, func_config, code_zip)+ # get_or_create_worker keys internally as f"{func_name}:{qualifier}",+ # so we must pass name + qualifier separately. Building a synthetic+ # `f"{func_name}:{qualifier}"` and passing it as func_name double-+ # suffixes the key (`fn:qual:$LATEST`), missing the warm pool the+ # SDK invoke path populates and forcing a cold start on every WS+ # message. That's why pre-warming the function via the SDK didn't+ # help WebSocket dispatch — keys didn't match.+ worker = get_or_create_worker(+ func_name, func_config, code_zip,+ qualifier=qualifier or "$LATEST",+ )result = await asyncio.to_thread(worker.invoke, event, message_id)if result.get("status") == "error":return {"statusCode": 500, "body": result.get("error", "")}diff --git a/ministack/services/transfer.py b/ministack/services/transfer.pyindex b806dc3f..95e50cff 100644--- a/ministack/services/transfer.py+++ b/ministack/services/transfer.py@@ -1,18 +1,46 @@"""AWS Transfer Family Service Emulator.-JSON-based API via X-Amz-Target: TransferService.<Operation>.-Supports:- Servers: CreateServer, DescribeServer, DeleteServer, ListServers- Users: CreateUser, DescribeUser, DeleteUser, ListUsers- SSH Keys: ImportSshPublicKey, DeleteSshPublicKey+Two protocols backing one AWS service:++- **Control plane** — JSON API via X-Amz-Target: TransferService.<Operation>.+ CreateServer / DescribeServer / DeleteServer / ListServers / StartServer /+ StopServer, CreateUser / DescribeUser / DeleteUser / ListUsers,+ ImportSshPublicKey / DeleteSshPublicKey.++- **Data plane** — real SFTP listener on :2222 by default+ (`SFTP_PORT`), key-authenticated, S3-backed virtual filesystem.+ `SFTP_PORT_PER_SERVER=1` opts into AWS-style per-server endpoints,+ allocating ports from `SFTP_BASE_PORT` (default 2300, incrementing).+ Authentication matches the presented SSH public key against every+ user's `SshPublicKeys` across every Transfer server (and across every+ account); the matching user determines the server identity, account,+ and home-directory mapping — no `username$serverid` decoration+ (that's a LocalStack-ism, not AWS behavior).++S3 semantics under SFTP+-----------------------+- `mkdir <prefix>` writes a zero-byte object at `<prefix>/`.+- `rename` is implemented as copy + delete (S3 has no atomic rename).+- Writes buffer in memory until close, then PUT the entire body at once.+- `setstat` / `chmod` / `chown` / `utime` are no-ops; S3 has no POSIX+ permissions or mtime.+- The host key persists at `${STATE_DIR}/transfer-host-key` when+ `PERSIST_STATE=1` (otherwise regenerated each run) so SSH clients+ don't see "host identification has changed" warnings between restarts."""+from __future__ import annotations++import asyncioimport copy+import ioimport jsonimport loggingimport osimport re+import time+from typing import AsyncIterator, Optionalfrom ministack.core.persistence import load_statefrom ministack.core.responses import (@@ -27,6 +55,21 @@ from ministack.core.responses import (logger = logging.getLogger("transfer")+# asyncssh is an optional extra (pip install ministack[full]). We import+# guarded so that a base install with the transfer service still works for+# the control plane (CreateServer / DescribeServer / etc.) even when the+# SFTP listener isn't available.+try:+ import asyncssh+ from asyncssh.sftp import SFTPAttrs, SFTPName++ _ASYNCSSH_AVAILABLE = True+except Exception: # noqa: BLE001 — any import failure means no SFTP+ asyncssh = None # type: ignore[assignment]+ SFTPAttrs = None # type: ignore[assignment]+ SFTPName = None # type: ignore[assignment]+ _ASYNCSSH_AVAILABLE = False+REGION = os.environ.get("MINISTACK_REGION", "us-east-1")# ---------------------------------------------------------------------------@@ -123,6 +166,8 @@ async def handle_request(method, path, headers, body, query_params):"DescribeServer": _describe_server,"DeleteServer": _delete_server,"ListServers": _list_servers,+ "StartServer": _start_server,+ "StopServer": _stop_server,"CreateUser": _create_user,"DescribeUser": _describe_user,"DeleteUser": _delete_user,@@ -134,14 +179,20 @@ async def handle_request(method, path, headers, body, query_params):handler = handlers.get(action)if not handler:return error_response_json("InvalidAction", f"Unknown action: {action}", 400)- return handler(data)+ result = handler(data)+ # Some handlers need to await SFTP listener setup; the dispatcher+ # transparently handles both sync and async return shapes so the+ # per-handler signature stays minimal.+ if asyncio.iscoroutine(result):+ result = await result+ return result# ---------------------------------------------------------------------------# Server handlers# ----------------------------------------------------------------------------def _create_server(data):+async def _create_server(data):sid = _server_id()now = now_iso()@@ -159,6 +210,17 @@ def _create_server(data):}_servers[sid] = server++ # If SFTP_PORT_PER_SERVER=1, allocate a dedicated SFTP listener for+ # this server (matches AWS's per-server endpoint model). Otherwise+ # the shared :2222 listener handles every server, disambiguating by+ # the user's SSH public key. Discovery of the port is via the admin+ # endpoint /_ministack/transfer/sftp-ports.+ try:+ await sftp_start_server_listener(sid)+ except Exception as e:+ logger.warning("SFTP per-server listener failed for %s: %s", sid, e)+return json_response({"ServerId": sid})@@ -180,10 +242,29 @@ def _describe_server(data):"Tags": server.get("Tags", []),"UserCount": server.get("UserCount", 0),}+ # SFTP port discovery is via GET /_ministack/transfer/sftp-ports —+ # boto3's model-driven deserializer drops fields not in the AWS spec+ # (so DescribeServer can't carry an SftpPort field meaningfully).return json_response({"Server": described})-def _delete_server(data):+def _start_server(data):+ sid = data.get("ServerId", "")+ if sid not in _servers:+ return _error("ResourceNotFoundException", f"Unknown server: {sid}", 404)+ _servers[sid]["State"] = "ONLINE"+ return json_response({})+++def _stop_server(data):+ sid = data.get("ServerId", "")+ if sid not in _servers:+ return _error("ResourceNotFoundException", f"Unknown server: {sid}", 404)+ _servers[sid]["State"] = "OFFLINE"+ return json_response({})+++async def _delete_server(data):sid = data.get("ServerId", "")if sid not in _servers:return _error("ResourceNotFoundException",@@ -195,6 +276,14 @@ def _delete_server(data):del _users[k]del _servers[sid]++ # Tear down any per-server SFTP listener bound for this server (no-op+ # in shared-port mode).+ try:+ await sftp_stop_server_listener(sid)+ except Exception as e:+ logger.debug("SFTP per-server stop error for %s: %s", sid, e)+return json_response({})@@ -407,3 +496,807 @@ def _delete_ssh_public_key(data):_users[uk]["SshPublicKeys"] = [k for k in keys if k["SshPublicKeyId"] != key_id]return json_response({})+++# ===========================================================================+# SFTP data plane+# ===========================================================================+#+# Lifecycle (called from app.py lifespan):+# - sftp_start() — bind shared listener on SFTP_PORT+# - sftp_stop() — close shared + per-server listeners+# - sftp_start_server_listener(sid) — per-server-port mode+# - sftp_stop_server_listener(sid) — per-server-port mode+# - sftp_per_server_port(sid) — discovery for the admin endpoint+#+# Everything else in this section is internal to the SFTP listener.+# ===========================================================================+++def _is_truthy(v):+ return (v or "").lower() in ("1", "true", "yes", "on")+++def _sftp_enabled() -> bool:+ if not _ASYNCSSH_AVAILABLE:+ return False+ raw = os.environ.get("SFTP_ENABLED")+ if raw is None:+ return True # default-on when asyncssh is installed+ return _is_truthy(raw)+++def _port_per_server() -> bool:+ return _is_truthy(os.environ.get("SFTP_PORT_PER_SERVER"))+++def _shared_port() -> int:+ return int(os.environ.get("SFTP_PORT", "2222"))+++def _per_server_base_port() -> int:+ return int(os.environ.get("SFTP_BASE_PORT", "2300"))+++def _bind_host() -> str:+ return os.environ.get("SFTP_HOST", "0.0.0.0")+++# Module-level state — listeners + host key+_sftp_shared_acceptor = None # type: Optional[object]+_sftp_per_server_acceptors: dict = {}+_sftp_per_server_ports: dict = {}+_sftp_next_per_server_port: Optional[int] = None+_sftp_host_key = None # type: Optional[object]+_sftp_lock = asyncio.Lock()+++def _sftp_host_key_path() -> str:+ from ministack.core.persistence import STATE_DIR+ return os.path.join(STATE_DIR, "transfer-host-key")+++def _sftp_load_or_generate_host_key():+ """Return an asyncssh-compatible host key.++ PERSIST_STATE=1 + key file exists → load it.+ PERSIST_STATE=1 + key file missing → generate, save, return.+ PERSIST_STATE=0 → generate ephemerally, no save.++ Persisting matters because OpenSSH clients cache the host key+ fingerprint in known_hosts; a fresh key on every container restart+ triggers the "REMOTE HOST IDENTIFICATION HAS CHANGED" warning that+ breaks automated SFTP clients.+ """+ global _sftp_host_key+ if _sftp_host_key is not None:+ return _sftp_host_key++ from ministack.core.persistence import PERSIST_STATE++ path = _sftp_host_key_path()+ if PERSIST_STATE and os.path.exists(path):+ try:+ _sftp_host_key = asyncssh.read_private_key(path)+ logger.info("SFTP: loaded persisted host key from %s", path)+ return _sftp_host_key+ except Exception as e:+ logger.warning("SFTP: failed to load persisted host key %s (%s); regenerating", path, e)++ _sftp_host_key = asyncssh.generate_private_key("ssh-ed25519")+ if PERSIST_STATE:+ try:+ os.makedirs(os.path.dirname(path), exist_ok=True)+ _sftp_host_key.write_private_key(path)+ logger.info("SFTP: persisted new host key at %s", path)+ except Exception as e:+ logger.warning("SFTP: failed to persist host key to %s: %s", path, e)+ else:+ logger.info("SFTP: generated ephemeral host key (PERSIST_STATE=0)")+ return _sftp_host_key+++def _sftp_normalize_key(key_body: str) -> str:+ """Strip comments + whitespace for stable comparison.++ SSH keys in `authorized_keys` look like ``ssh-ed25519 AAAAC3...== comment``.+ The comment is informational; the key payload is the second field.+ """+ parts = key_body.strip().split()+ if len(parts) >= 2:+ return f"{parts[0]} {parts[1]}"+ return key_body.strip()+++def _sftp_resolve_user_by_key(username: str, presented_key_body: str):+ """Scan every Transfer user across every account for a matching+ (UserName, SshPublicKey) pair. Returns ``(account_id, server_id, user)``+ or ``None``.++ AccountScopedDict stores entries under tuple keys ``(account_id, key)``,+ so we iterate ``_data.items()`` directly to scan across accounts —+ `__iter__` would only yield the current request's account, which isn't+ set during SFTP auth.+ """+ presented = _sftp_normalize_key(presented_key_body)+ for scoped_key, user in _users._data.items():+ account_id, _ = scoped_key+ if user.get("UserName") != username:+ continue+ for key_record in user.get("SshPublicKeys", []):+ if _sftp_normalize_key(key_record.get("SshPublicKeyBody", "")) == presented:+ return account_id, user["ServerId"], user+ return None+++def _sftp_server_state(account_id: str, server_id: str):+ """Return the State of a Transfer server (ONLINE / OFFLINE) or None+ if the server doesn't exist for the given account."""+ server = _servers._data.get((account_id, server_id))+ return server["State"] if server else None+++# ── S3 path resolution (HomeDirectory + HomeDirectoryMappings) ─────────────+++def _sftp_normalize_virtual_path(virtual: str) -> str:+ """Make `virtual` an absolute, slash-rooted path with no trailing slash+ (except for the root). Resolves `..` to prevent chroot escape."""+ if not virtual:+ virtual = "/"+ if not virtual.startswith("/"):+ virtual = "/" + virtual+ parts: list = []+ for seg in virtual.split("/"):+ if seg in ("", "."):+ continue+ if seg == "..":+ if parts:+ parts.pop()+ continue+ parts.append(seg)+ if not parts:+ return "/"+ return "/" + "/".join(parts)+++def _sftp_resolve_s3_target(user: dict, virtual_path: str):+ """Translate an SFTP-side virtual path to an S3 ``(bucket, key)`` pair.++ PATH mode — `HomeDirectory` is e.g. `/my-bucket/some/prefix`.+ LOGICAL — `HomeDirectoryMappings` list, longest prefix wins.+ """+ virtual_path = _sftp_normalize_virtual_path(virtual_path)+ home_type = user.get("HomeDirectoryType", "PATH")++ if home_type == "LOGICAL":+ mappings = user.get("HomeDirectoryMappings") or []+ sorted_maps = sorted(mappings, key=lambda m: len(m.get("Entry", "")), reverse=True)+ for mapping in sorted_maps:+ entry = _sftp_normalize_virtual_path(mapping.get("Entry", "/"))+ target = mapping.get("Target", "")+ if virtual_path == entry or virtual_path.startswith(entry + "/"):+ rest = virtual_path[len(entry):].lstrip("/")+ full = target.rstrip("/") + ("/" + rest if rest else "")+ return _sftp_split_bucket_key(full)+ return "", virtual_path.lstrip("/")++ # PATH mode+ home = user.get("HomeDirectory") or "/"+ base = _sftp_normalize_virtual_path(home)+ if virtual_path == "/":+ full = base+ elif virtual_path.startswith(base + "/") or virtual_path == base:+ full = virtual_path+ else:+ full = base.rstrip("/") + virtual_path+ return _sftp_split_bucket_key(full)+++def _sftp_split_bucket_key(full: str):+ """Split ``/bucket/key/segments`` → ``("bucket", "key/segments")``."""+ full = full.lstrip("/")+ if not full:+ return "", ""+ if "/" in full:+ bucket, key = full.split("/", 1)+ return bucket, key+ return full, ""+++# ── S3 helpers — read / write / list against the in-memory S3 service ──────+++def _sftp_s3_bucket(account_id: str, bucket: str):+ """Return the bucket dict for the given account, or None.++ S3 lives in a different module so we lazy-import inside the helper to+ avoid an import cycle at module load.+ """+ from ministack.services import s3 as s3_mod+ return s3_mod._buckets._data.get((account_id, bucket))+++def _sftp_s3_list_keys(account_id: str, bucket: str, prefix: str):+ b = _sftp_s3_bucket(account_id, bucket)+ if not b:+ return []+ return [k for k in b.get("objects", {}).keys() if k.startswith(prefix)]+++def _sftp_s3_get_object(account_id: str, bucket: str, key: str):+ b = _sftp_s3_bucket(account_id, bucket)+ if not b:+ return None+ return b.get("objects", {}).get(key)+++def _sftp_s3_put_object(account_id: str, bucket: str, key: str, body: bytes) -> None:+ """Write a new object body using the canonical S3 record schema so+ subsequent GetObject / HeadObject calls via the standard S3 API+ return the bytes unchanged. Mirrors ``s3._build_object_record``."""+ import hashlib++ b = _sftp_s3_bucket(account_id, bucket)+ if not b:+ raise FileNotFoundError(f"bucket {bucket} not found")+ objects = b.setdefault("objects", {})+ objects[key] = {+ "body": body,+ "content_type": "application/octet-stream",+ "content_encoding": None,+ "etag": '"' + hashlib.md5(body, usedforsecurity=False).hexdigest() + '"',+ "last_modified": now_iso(),+ "size": len(body),+ "metadata": {},+ "preserved_headers": {},+ }+++def _sftp_s3_delete_object(account_id: str, bucket: str, key: str) -> bool:+ b = _sftp_s3_bucket(account_id, bucket)+ if not b:+ return False+ objects = b.get("objects", {})+ if key in objects:+ del objects[key]+ return True+ return False+++# ── SFTP / SSH server classes ──────────────────────────────────────────────+++def _sftp_build_sftp_factory(restrict_to_server_id: Optional[str] = None):+ """asyncssh sftp_factory closure. When ``restrict_to_server_id`` is set+ (per-server-port mode), the SFTP layer rejects connections whose+ authenticated user belongs to a different Transfer server."""+ def _factory(chan):+ return _MiniStackSFTPServer(chan, restrict_to_server_id=restrict_to_server_id)+ return _factory+++def _sftp_build_server_factory(restrict_to_server_id: Optional[str] = None):+ """asyncssh `SSHServer` factory closure."""+ def _factory():+ return _MiniStackSSHServer(restrict_to_server_id=restrict_to_server_id)+ return _factory+++class _MiniStackSSHServer(asyncssh.SSHServer if _ASYNCSSH_AVAILABLE else object):+ """SSH-side auth callback. Resolves the (account, server, user) tuple+ from the presented public key and stashes it on the connection so the+ SFTP layer can read it back.++ Subclassing :class:`asyncssh.SSHServer` (rather than just duck-typing+ the relevant methods) is required because asyncssh probes a wide set+ of optional auth methods (host-based, kbdint, GSSAPI) on the server+ instance during userauth negotiation; the default base-class+ implementations return ``False`` for everything we don't support.+ """++ def __init__(self, restrict_to_server_id: Optional[str] = None):+ super().__init__() if _ASYNCSSH_AVAILABLE else None+ self._restrict_to_server_id = restrict_to_server_id+ self._resolved = None+ self._username = None+ self._conn = None++ def connection_made(self, conn):+ self._conn = conn++ def connection_lost(self, exc):+ pass++ def begin_auth(self, username):+ # True → continue to public-key validation. False would mean "no+ # auth needed", which we never want.+ self._username = username+ return True++ def password_auth_supported(self):+ return False++ def public_key_auth_supported(self):+ return True++ def validate_public_key(self, username, key):+ """Called by asyncssh for each presented public key. Returns True+ if the key is registered for this username on some Transfer server.++ We export the key in OpenSSH format and string-compare against the+ body stored via `ImportSshPublicKey`. This is deliberately stricter+ than fingerprint comparison — same key body, exact match.+ """+ try:+ presented_body = key.export_public_key("openssh").decode("utf-8")+ except Exception as e:+ logger.debug("SFTP: failed to export presented key: %s", e)+ return False++ match = _sftp_resolve_user_by_key(username, presented_body)+ if not match:+ return False++ account_id, server_id, user = match+ if self._restrict_to_server_id and server_id != self._restrict_to_server_id:+ return False+ if _sftp_server_state(account_id, server_id) != "ONLINE":+ return False++ self._resolved = (account_id, server_id, user)+ if self._conn is not None:+ self._conn.set_extra_info(ministack_user=user)+ self._conn.set_extra_info(ministack_server_id=server_id)+ self._conn.set_extra_info(ministack_account_id=account_id)+ return True+++class _MiniStackSFTPServer(asyncssh.SFTPServer if _ASYNCSSH_AVAILABLE else object):+ """SFTP filesystem backed by MiniStack's in-memory S3 state.++ asyncssh instantiates this per session, passing the SSH channel so we+ can recover the auth metadata stashed by `_MiniStackSSHServer`.++ Subclassing :class:`asyncssh.SFTPServer` (rather than duck-typing) is+ required so unsupported file ops (statvfs, link, etc.) inherit the+ base implementations that politely return SSH_FX_OP_UNSUPPORTED to+ the client instead of crashing the session.+ """++ def __init__(self, chan, restrict_to_server_id: Optional[str] = None):+ if _ASYNCSSH_AVAILABLE:+ super().__init__(chan)+ self._chan = chan+ self._restrict_to_server_id = restrict_to_server_id+ # asyncssh's SSHServerChannel exposes the parent connection via+ # ``get_connection``; older versions expose it as ``_conn``.+ conn = None+ if hasattr(chan, "get_connection"):+ conn = chan.get_connection()+ elif hasattr(chan, "_conn"):+ conn = chan._conn+ self._user = conn.get_extra_info("ministack_user") if conn else None+ self._account_id = conn.get_extra_info("ministack_account_id") if conn else None+ self._server_id = conn.get_extra_info("ministack_server_id") if conn else None+ # Open-file map. Reads use BytesIO over the existing object body;+ # writes accumulate into a BytesIO that we PUT on close.+ self._open_files: dict = {}+ self._next_handle = 1++ def _resolve(self, path):+ """bytes path → (bucket, key) for the currently authenticated user,+ within their account context."""+ from ministack.core.responses import set_request_account_id++ if self._account_id:+ set_request_account_id(self._account_id)+ virtual = path.decode("utf-8", errors="replace") if isinstance(path, bytes) else path+ return _sftp_resolve_s3_target(self._user or {}, virtual)++ @staticmethod+ def _attrs_for_file(size, mtime):+ a = SFTPAttrs()+ a.type = 1 # SSH_FILEXFER_TYPE_REGULAR+ a.size = size+ a.uid = 0+ a.gid = 0+ a.permissions = 0o100644+ a.atime = mtime+ a.mtime = mtime+ return a++ @staticmethod+ def _attrs_for_dir(mtime):+ a = SFTPAttrs()+ a.type = 2 # SSH_FILEXFER_TYPE_DIRECTORY+ a.size = 0+ a.uid = 0+ a.gid = 0+ a.permissions = 0o040755+ a.atime = mtime+ a.mtime = mtime+ return a++ # ---- path / metadata ops --------------------------------------------++ def realpath(self, path):+ virtual = path.decode("utf-8", errors="replace")+ return _sftp_normalize_virtual_path(virtual).encode("utf-8")++ def stat(self, path):+ bucket, key = self._resolve(path)+ if not bucket:+ return self._attrs_for_dir(int(time.time()))+ obj = _sftp_s3_get_object(self._account_id, bucket, key)+ if obj:+ return self._attrs_for_file(obj.get("size", 0), int(time.time()))+ prefix = key.rstrip("/") + "/" if key else ""+ for _ in _sftp_s3_list_keys(self._account_id, bucket, prefix):+ return self._attrs_for_dir(int(time.time()))+ if _sftp_s3_bucket(self._account_id, bucket) is None:+ raise asyncssh.SFTPNoSuchFile(f"No such bucket: {bucket}")+ raise asyncssh.SFTPNoSuchFile(f"No such file: {path!r}")++ def lstat(self, path):+ return self.stat(path)++ def fstat(self, file_obj):+ h = self._open_files.get(file_obj)+ if not h:+ raise asyncssh.SFTPFailure("Bad file handle")+ size = h["size"] if h["mode"] == "read" else len(h["buf"].getvalue())+ return self._attrs_for_file(size, h.get("mtime", int(time.time())))++ def setstat(self, path, attrs):+ # No-op so `sftp -p` and rsync don't fail on the mode/time copy.+ return None++ def fsetstat(self, file_obj, attrs):+ return None++ def lsetstat(self, path, attrs):+ return None++ # ---- directory ops ---------------------------------------------------++ async def scandir(self, path):+ bucket, key = self._resolve(path)+ seen_names: set = set()++ for special in (".", ".."):+ attrs = self._attrs_for_dir(int(time.time()))+ yield SFTPName(+ filename=special.encode("utf-8"),+ longname=self._format_longname(special, attrs).encode("utf-8"),+ attrs=attrs,+ )++ if not bucket:+ virtual = path.decode("utf-8", errors="replace") if isinstance(path, bytes) else path+ virtual = _sftp_normalize_virtual_path(virtual)+ sub_entries: set = set()+ for mapping in (self._user or {}).get("HomeDirectoryMappings", []) or []:+ entry = _sftp_normalize_virtual_path(mapping.get("Entry", "/"))+ if entry == virtual:+ continue+ if entry.startswith(virtual.rstrip("/") + "/") or virtual == "/":+ suffix = entry[len(virtual.rstrip("/")) + 1:].split("/")[0]+ if suffix:+ sub_entries.add(suffix)+ for name in sorted(sub_entries):+ if name in seen_names:+ continue+ seen_names.add(name)+ attrs = self._attrs_for_dir(int(time.time()))+ yield SFTPName(+ filename=name.encode("utf-8"),+ longname=self._format_longname(name, attrs).encode("utf-8"),+ attrs=attrs,+ )+ return++ prefix = key.rstrip("/") + "/" if key else ""+ b = _sftp_s3_bucket(self._account_id, bucket)+ if b is None:+ return++ for full_key in sorted(b.get("objects", {}).keys()):+ if not full_key.startswith(prefix):+ continue+ remainder = full_key[len(prefix):]+ if not remainder:+ continue+ head, sep, _ = remainder.partition("/")+ if sep:+ if head in seen_names:+ continue+ seen_names.add(head)+ attrs = self._attrs_for_dir(int(time.time()))+ yield SFTPName(+ filename=head.encode("utf-8"),+ longname=self._format_longname(head, attrs).encode("utf-8"),+ attrs=attrs,+ )+ else:+ if head in seen_names:+ continue+ seen_names.add(head)+ obj = b["objects"][full_key]+ attrs = self._attrs_for_file(obj.get("size", 0), int(time.time()))+ yield SFTPName(+ filename=head.encode("utf-8"),+ longname=self._format_longname(head, attrs).encode("utf-8"),+ attrs=attrs,+ )++ def _format_longname(self, name, attrs) -> str:+ """ls-l-style line for asyncssh's longname field."""+ is_dir = attrs.type == 2+ mode_str = "drwxr-xr-x" if is_dir else "-rw-r--r--"+ size = attrs.size or 0+ mtime = attrs.mtime or int(time.time())+ ts = time.strftime("%b %d %H:%M", time.localtime(mtime))+ return f"{mode_str} 1 ministack ministack {size:>10d} {ts} {name}"++ def mkdir(self, path, attrs):+ bucket, key = self._resolve(path)+ if not bucket:+ raise asyncssh.SFTPFailure("Cannot create directory above bucket level")+ if not key:+ return None+ placeholder_key = key.rstrip("/") + "/"+ _sftp_s3_put_object(self._account_id, bucket, placeholder_key, b"")+ return None++ def rmdir(self, path):+ bucket, key = self._resolve(path)+ if not bucket or not key:+ raise asyncssh.SFTPFailure("Cannot remove bucket root via SFTP")+ prefix = key.rstrip("/") + "/"+ children = [k for k in _sftp_s3_list_keys(self._account_id, bucket, prefix) if k != prefix]+ if children:+ raise asyncssh.SFTPFailure(f"Directory not empty: {path!r}")+ _sftp_s3_delete_object(self._account_id, bucket, prefix)+ return None++ def remove(self, path):+ bucket, key = self._resolve(path)+ if not bucket or not key:+ raise asyncssh.SFTPNoSuchFile(f"Cannot remove {path!r}")+ if not _sftp_s3_delete_object(self._account_id, bucket, key):+ raise asyncssh.SFTPNoSuchFile(f"No such file: {path!r}")+ return None++ def rename(self, oldpath, newpath):+ return self._do_rename(oldpath, newpath, allow_overwrite=False)++ def posix_rename(self, oldpath, newpath):+ return self._do_rename(oldpath, newpath, allow_overwrite=True)++ def _do_rename(self, oldpath, newpath, *, allow_overwrite: bool):+ src_bucket, src_key = self._resolve(oldpath)+ dst_bucket, dst_key = self._resolve(newpath)+ if not src_bucket or not src_key:+ raise asyncssh.SFTPNoSuchFile(f"No such file: {oldpath!r}")+ src_obj = _sftp_s3_get_object(self._account_id, src_bucket, src_key)+ if src_obj is None:+ raise asyncssh.SFTPNoSuchFile(f"No such file: {oldpath!r}")+ if not dst_bucket:+ raise asyncssh.SFTPFailure("Cannot rename above bucket level")+ if not allow_overwrite and _sftp_s3_get_object(self._account_id, dst_bucket, dst_key):+ raise asyncssh.SFTPFailure(f"Destination exists: {newpath!r}")+ # S3 has no atomic rename — copy + delete.+ _sftp_s3_put_object(self._account_id, dst_bucket, dst_key, src_obj.get("body", b""))+ _sftp_s3_delete_object(self._account_id, src_bucket, src_key)+ return None++ # ---- file ops --------------------------------------------------------++ def open(self, path, pflags, attrs):+ # asyncssh masks: SSH_FXF_READ=0x01, SSH_FXF_WRITE=0x02,+ # SSH_FXF_APPEND=0x04, SSH_FXF_CREAT=0x08, SSH_FXF_TRUNC=0x10,+ # SSH_FXF_EXCL=0x20.+ bucket, key = self._resolve(path)+ if not bucket or not key:+ raise asyncssh.SFTPFailure("Open requires bucket+key")++ wants_write = bool(pflags & 0x02)+ wants_create = bool(pflags & 0x08)+ wants_truncate = bool(pflags & 0x10)+ wants_excl = bool(pflags & 0x20)++ existing = _sftp_s3_get_object(self._account_id, bucket, key)++ if wants_write:+ if existing and wants_excl:+ raise asyncssh.SFTPFailure(f"File exists: {path!r}")+ if not existing and not wants_create:+ raise asyncssh.SFTPNoSuchFile(f"No such file: {path!r}")+ initial = b"" if (wants_truncate or not existing) else existing.get("body", b"")+ handle = self._next_handle+ self._next_handle += 1+ self._open_files[handle] = {+ "mode": "write",+ "bucket": bucket,+ "key": key,+ "buf": io.BytesIO(initial),+ "size": len(initial),+ "mtime": int(time.time()),+ }+ return handle++ # Read mode+ if not existing:+ raise asyncssh.SFTPNoSuchFile(f"No such file: {path!r}")+ handle = self._next_handle+ self._next_handle += 1+ body = existing.get("body", b"")+ self._open_files[handle] = {+ "mode": "read",+ "bucket": bucket,+ "key": key,+ "buf": io.BytesIO(body),+ "size": len(body),+ "mtime": int(time.time()),+ }+ return handle++ def read(self, file_obj, offset, size):+ h = self._open_files.get(file_obj)+ if not h:+ raise asyncssh.SFTPFailure("Bad file handle")+ h["buf"].seek(offset)+ data = h["buf"].read(size)+ return data if data else b""++ def write(self, file_obj, offset, data):+ h = self._open_files.get(file_obj)+ if not h:+ raise asyncssh.SFTPFailure("Bad file handle")+ if h["mode"] != "write":+ raise asyncssh.SFTPFailure("File not open for writing")+ h["buf"].seek(offset)+ h["buf"].write(data)+ return len(data)++ def close(self, file_obj):+ h = self._open_files.pop(file_obj, None)+ if not h:+ return None+ if h["mode"] == "write":+ _sftp_s3_put_object(self._account_id, h["bucket"], h["key"], h["buf"].getvalue())+ return None++ def fsync(self, file_obj):+ return None++ # ---- explicitly unsupported -----------------------------------------++ def link(self, oldpath, newpath):+ raise asyncssh.SFTPOpUnsupported("S3-backed SFTP does not support hardlinks")++ def symlink(self, oldpath, newpath):+ raise asyncssh.SFTPOpUnsupported("S3-backed SFTP does not support symlinks")++ def readlink(self, path):+ raise asyncssh.SFTPOpUnsupported("S3-backed SFTP does not support symlinks")++ def lock(self, file_obj, offset, length, flags):+ return None++ def unlock(self, file_obj, offset, length):+ return None++ def exit(self):+ # Flush handles the client forgot to close.+ for handle in list(self._open_files):+ self.close(handle)+++# ── SFTP lifecycle — called from app.py lifespan + Create/Delete handlers ──+++async def sftp_start() -> None:+ """Idempotent: bind the shared SFTP listener on `SFTP_PORT` if enabled."""+ if not _sftp_enabled():+ logger.info("SFTP: disabled (asyncssh missing or SFTP_ENABLED=0)")+ return++ global _sftp_shared_acceptor, _sftp_next_per_server_port+ _sftp_next_per_server_port = _per_server_base_port()++ async with _sftp_lock:+ if _sftp_shared_acceptor is not None:+ return+ host_key = _sftp_load_or_generate_host_key()+ port = _shared_port()+ try:+ _sftp_shared_acceptor = await asyncssh.listen(+ host=_bind_host(),+ port=port,+ server_factory=_sftp_build_server_factory(),+ server_host_keys=[host_key],+ sftp_factory=_sftp_build_sftp_factory(),+ allow_scp=False,+ )+ logger.info("SFTP: listening on %s:%d (shared)", _bind_host(), port)+ except OSError as e:+ logger.warning("SFTP: failed to bind %s:%d (%s); SFTP unavailable", _bind_host(), port, e)+ _sftp_shared_acceptor = None+++async def sftp_stop() -> None:+ """Stop the shared listener and any per-server listeners."""+ global _sftp_shared_acceptor+ async with _sftp_lock:+ if _sftp_shared_acceptor is not None:+ try:+ _sftp_shared_acceptor.close()+ await _sftp_shared_acceptor.wait_closed()+ except Exception as e:+ logger.debug("SFTP: error closing shared acceptor: %s", e)+ _sftp_shared_acceptor = None+ for sid, acceptor in list(_sftp_per_server_acceptors.items()):+ try:+ acceptor.close()+ await acceptor.wait_closed()+ except Exception as e:+ logger.debug("SFTP: error closing per-server acceptor for %s: %s", sid, e)+ _sftp_per_server_acceptors.clear()+ _sftp_per_server_ports.clear()+ logger.info("SFTP: stopped")+++async def sftp_start_server_listener(server_id: str) -> Optional[int]:+ """Bind a per-server SFTP port for `server_id`. Returns the port (or+ None when SFTP_PORT_PER_SERVER is off or asyncssh isn't available).++ Called from `_create_server` when per-server-port mode is on.+ """+ if not (_sftp_enabled() and _port_per_server()):+ return None+ global _sftp_next_per_server_port+ if _sftp_next_per_server_port is None:+ _sftp_next_per_server_port = _per_server_base_port()+ async with _sftp_lock:+ if server_id in _sftp_per_server_acceptors:+ return _sftp_per_server_ports.get(server_id)+ host_key = _sftp_load_or_generate_host_key()+ port = _sftp_next_per_server_port+ _sftp_next_per_server_port += 1+ try:+ acceptor = await asyncssh.listen(+ host=_bind_host(),+ port=port,+ server_factory=_sftp_build_server_factory(server_id),+ server_host_keys=[host_key],+ sftp_factory=_sftp_build_sftp_factory(server_id),+ allow_scp=False,+ )+ _sftp_per_server_acceptors[server_id] = acceptor+ _sftp_per_server_ports[server_id] = port+ logger.info("SFTP: per-server listener for %s on %s:%d", server_id, _bind_host(), port)+ return port+ except OSError as e:+ logger.warning("SFTP: failed to bind per-server port %d for %s: %s", port, server_id, e)+ return None+++async def sftp_stop_server_listener(server_id: str) -> None:+ async with _sftp_lock:+ acceptor = _sftp_per_server_acceptors.pop(server_id, None)+ _sftp_per_server_ports.pop(server_id, None)+ if acceptor is None:+ return+ try:+ acceptor.close()+ await acceptor.wait_closed()+ except Exception as e:+ logger.debug("SFTP: error closing per-server acceptor for %s: %s", server_id, e)+++def sftp_per_server_port(server_id: str) -> Optional[int]:+ """Return the bound port for `server_id` in per-server-port mode, if any.+ Used by the `/_ministack/transfer/sftp-ports` admin endpoint."""+ return _sftp_per_server_ports.get(server_id)diff --git a/pyproject.toml b/pyproject.tomlindex 647f9d3f..56541cbd 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -34,6 +34,7 @@ full = ["cryptography>=41.0","psycopg2-binary>=2.9","pymysql>=1.1",+ "asyncssh>=2.14",]dev = ["boto3>=1.34",@@ -47,6 +48,7 @@ dev = ["pymysql>=1.1","ruff>=0.4","mypy-boto3-lambda (>=1.42.85,<2.0.0)",+ "asyncssh>=2.14",][project.urls]diff --git a/tests/conftest.py b/tests/conftest.pyindex f7017a78..3f2bf6f2 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -44,6 +44,18 @@ _SERIAL_TESTS = {"tests/test_sfn.py::test_sfn_wait_scale_zero_skips_wait","tests/test_rds_lambda_network.py::test_rds_lambda_network_connectivity","tests/test_elasticache_lambda_network.py::test_elasticache_lambda_network_connectivity",+ # API Gateway execute-api → Lambda invoke under tight urlopen / WS recv+ # timeouts. These pass cleanly when run serially but are sensitive to+ # xdist parallel load on shared CI runners (cold-start time bursts past+ # the client timeout). Pre-warming + the WS warm-pool key fix covered+ # the deterministic cases; these remaining ones tip over only under+ # sustained parallel pressure. Run them in the dedicated serial phase.+ "tests/test_apigatewayv2.py::test_apigwv2_path_based_execute_api_http",+ "tests/test_apigatewayv2.py::test_apigwv2_path_based_websocket",+ "tests/test_apigatewayv2.py::test_apigwv2_default_stage_serves_from_root",+ "tests/test_apigatewayv2.py::test_apigwv1_path_based_restapi_legacy_user_request",+ "tests/test_apigatewayv2.py::test_apigwv2_named_stage_still_requires_prefix",+ "tests/test_apigatewayv2.py::test_apigwv2_integration_wrapped_function_arn",}diff --git a/tests/test_apigatewayv2.py b/tests/test_apigatewayv2.pyindex c811fa59..173d12c2 100644--- a/tests/test_apigatewayv2.py+++ b/tests/test_apigatewayv2.py@@ -1039,6 +1039,22 @@ def _make_fn(lam, name: str, code: str) -> str:Handler="index.handler",Code={"ZipFile": _make_zip(code)},)+ # Pre-warm the warm-pool subprocess so subsequent execute-api invocations+ # don't pay the zip-extract + interpreter-boot cost on the first request.+ # boto3's invoke has a generous read timeout (60s) and tolerates spawn+ # latency that the apigwv2 tests' 5-second urlopen does not — under xdist+ # parallel load on shared CI runners the cold start can exceed 5s and+ # cause opaque socket timeouts (#404-class flakes).+ # Handler errors on the warmup payload are non-fatal: the subprocess is+ # warm regardless, which is the only thing we need.+ try:+ lam.invoke(+ FunctionName=name,+ InvocationType="RequestResponse",+ Payload=b'{"_ministack_warmup": true}',+ )+ except Exception:+ passreturn f"arn:aws:lambda:us-east-1:000000000000:function:{name}"diff --git a/tests/test_transfer_sftp.py b/tests/test_transfer_sftp.pynew file mode 100644index 00000000..77c6835e--- /dev/null+++ b/tests/test_transfer_sftp.py@@ -0,0 +1,437 @@+"""+Integration tests for the Transfer Family SFTP layer.++These tests require the MiniStack server to be running (the standard+``MINISTACK_ENDPOINT`` env var picks up the address). They also require+``asyncssh`` to be importable — if the host's MiniStack process started+without asyncssh installed, the SFTP listener won't be up and every test+in this file is skipped.++Coverage beyond the happy path:+- Wrong key rejected, unknown user rejected, OFFLINE server rejected.+- Rename across prefixes (S3 copy + delete).+- Large file (> 1 MiB) round-trip.+- Concurrent uploads from two distinct users.+- LOGICAL HomeDirectoryMappings: virtual `/inbox` → bucket prefix.+- mkdir + listing + rmdir against the S3-prefix VFS.+- Per-server-port mode (skipped unless ``SFTP_PORT_PER_SERVER=1`` is set+ on the running server).+"""++import asyncio+import os+import socket+import time+import uuid++import pytest+import urllib.request++asyncssh = pytest.importorskip("asyncssh")+from cryptography.hazmat.primitives import serialization+from cryptography.hazmat.primitives.asymmetric import ed25519+++ENDPOINT_HOST = os.environ.get("MINISTACK_HOST", "127.0.0.1")+SFTP_PORT = int(os.environ.get("SFTP_PORT", "2222"))++# Honour MINISTACK_ENDPOINT (e.g. http://localhost:14566) so the same test+# file works against a locally-built MiniStack on port 4566 *and* against a+# preview Docker image bound to a different host port.+_endpoint = os.environ.get("MINISTACK_ENDPOINT", f"http://{ENDPOINT_HOST}:4566")+ADMIN_BASE = _endpoint.rstrip("/")+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _gen_keypair():+ """Return ``(private_pem_text, public_openssh_text)`` for an ed25519 key.++ asyncssh's ``connect(client_keys=...)`` accepts in-memory PEM, so we+ avoid touching the filesystem.+ """+ priv = ed25519.Ed25519PrivateKey.generate()+ priv_pem = priv.private_bytes(+ encoding=serialization.Encoding.PEM,+ format=serialization.PrivateFormat.OpenSSH,+ encryption_algorithm=serialization.NoEncryption(),+ ).decode("utf-8")+ pub = priv.public_key().public_bytes(+ encoding=serialization.Encoding.OpenSSH,+ format=serialization.PublicFormat.OpenSSH,+ ).decode("utf-8")+ return priv_pem, pub+++def _sftp_listening() -> bool:+ """Best-effort probe: return True if something is accepting TCP on+ SFTP_PORT. Skips the whole module if the listener isn't up."""+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)+ s.settimeout(1.0)+ try:+ s.connect((ENDPOINT_HOST, SFTP_PORT))+ except Exception:+ return False+ finally:+ s.close()+ return True+++pytestmark = pytest.mark.skipif(+ not _sftp_listening(),+ reason=f"SFTP listener not reachable at {ENDPOINT_HOST}:{SFTP_PORT} — start MiniStack first",+)+++def _connect(username, priv_pem, port=None):+ """Return an asyncssh connection coroutine for the given creds."""+ return asyncssh.connect(+ host=ENDPOINT_HOST,+ port=port or SFTP_PORT,+ username=username,+ client_keys=[asyncssh.import_private_key(priv_pem)],+ known_hosts=None,+ )+++def _provision(transfer, s3, *, bucket=None, home_prefix="", logical_mappings=None):+ """CreateServer + CreateBucket + CreateUser. Returns the dict the+ individual tests need: server_id, user_name, priv_pem, bucket name.+ """+ suffix = uuid.uuid4().hex[:8]+ server_id = transfer.create_server()["ServerId"]+ user_name = f"u-{suffix}"+ bucket = bucket or f"sftp-bucket-{suffix}"+ s3.create_bucket(Bucket=bucket)+ priv_pem, pub_ssh = _gen_keypair()++ user_kwargs = {+ "ServerId": server_id,+ "UserName": user_name,+ "Role": f"arn:aws:iam::000000000000:role/sftp-{suffix}",+ "SshPublicKeyBody": pub_ssh,+ }+ if logical_mappings is not None:+ user_kwargs["HomeDirectoryType"] = "LOGICAL"+ user_kwargs["HomeDirectoryMappings"] = logical_mappings+ else:+ user_kwargs["HomeDirectoryType"] = "PATH"+ user_kwargs["HomeDirectory"] = f"/{bucket}/{home_prefix}".rstrip("/")++ transfer.create_user(**user_kwargs)+ return {+ "server_id": server_id,+ "user_name": user_name,+ "priv_pem": priv_pem,+ "bucket": bucket,+ "describe": transfer.describe_server(ServerId=server_id)["Server"],+ }+++# ---------------------------------------------------------------------------+# Happy path+# ---------------------------------------------------------------------------+++def test_sftp_basic_upload_download(transfer, s3):+ """Connect, upload a file via SFTP, verify it landed in S3, download+ it back. End-to-end smoke test."""+ ctx = _provision(transfer, s3)++ async def _run():+ async with await _connect(ctx["user_name"], ctx["priv_pem"]) as conn:+ async with conn.start_sftp_client() as sftp:+ async with sftp.open("/hello.txt", "wb") as f:+ await f.write(b"hello sftp\n")+ # Re-list to confirm the file appears.+ names = await sftp.listdir("/")+ assert "hello.txt" in [n for n in names]+ # Download and verify.+ async with sftp.open("/hello.txt", "rb") as f:+ body = await f.read()+ assert body == b"hello sftp\n"++ asyncio.run(_run())++ # Verify it actually landed in S3 via the standard API.+ obj = s3.get_object(Bucket=ctx["bucket"], Key="hello.txt")+ assert obj["Body"].read() == b"hello sftp\n"+ transfer.delete_server(ServerId=ctx["server_id"])+++# ---------------------------------------------------------------------------+# Auth-failure cases+# ---------------------------------------------------------------------------+++def test_sftp_wrong_key_rejected(transfer, s3):+ """A connecting client with a public key that wasn't ImportSshPublicKey'd+ for the user must be rejected — not silently allowed in."""+ ctx = _provision(transfer, s3)+ _other_priv, _ = _gen_keypair()+ other_priv_pem, _ = _gen_keypair()++ async def _run():+ with pytest.raises((asyncssh.PermissionDenied, asyncssh.misc.PermissionDenied)):+ await _connect(ctx["user_name"], other_priv_pem)++ asyncio.run(_run())+ transfer.delete_server(ServerId=ctx["server_id"])+++def test_sftp_unknown_user_rejected(transfer, s3):+ """A username with no matching Transfer user must be rejected."""+ ctx = _provision(transfer, s3)++ async def _run():+ with pytest.raises((asyncssh.PermissionDenied, asyncssh.misc.PermissionDenied)):+ await _connect("nobody-" + uuid.uuid4().hex[:8], ctx["priv_pem"])++ asyncio.run(_run())+ transfer.delete_server(ServerId=ctx["server_id"])+++def test_sftp_offline_server_rejects_auth(transfer, s3):+ """StopServer flips State=OFFLINE; the SFTP listener must refuse+ auth for that server's users (matches AWS — a stopped server doesn't+ serve traffic). Bringing it back ONLINE re-allows auth."""+ ctx = _provision(transfer, s3)+ transfer.stop_server(ServerId=ctx["server_id"])++ async def _denied():+ with pytest.raises((asyncssh.PermissionDenied, asyncssh.misc.PermissionDenied)):+ await _connect(ctx["user_name"], ctx["priv_pem"])++ asyncio.run(_denied())++ transfer.start_server(ServerId=ctx["server_id"])++ async def _allowed():+ async with await _connect(ctx["user_name"], ctx["priv_pem"]) as conn:+ async with conn.start_sftp_client() as sftp:+ async with sftp.open("/probe", "wb") as f:+ await f.write(b"x")++ asyncio.run(_allowed())+ transfer.delete_server(ServerId=ctx["server_id"])+++# ---------------------------------------------------------------------------+# VFS / S3 semantics+# ---------------------------------------------------------------------------+++def test_sftp_rename_across_prefixes(transfer, s3):+ """Rename should work across nested prefixes — implemented as S3+ copy + delete since S3 has no atomic rename."""+ ctx = _provision(transfer, s3)++ async def _run():+ async with await _connect(ctx["user_name"], ctx["priv_pem"]) as conn:+ async with conn.start_sftp_client() as sftp:+ async with sftp.open("/a.txt", "wb") as f:+ await f.write(b"alpha")+ await sftp.rename("/a.txt", "/sub/b.txt")+ async with sftp.open("/sub/b.txt", "rb") as f:+ assert await f.read() == b"alpha"+ # Source must be gone.+ with pytest.raises(asyncssh.SFTPError):+ await sftp.stat("/a.txt")++ asyncio.run(_run())+ transfer.delete_server(ServerId=ctx["server_id"])+++def test_sftp_large_file_roundtrip(transfer, s3):+ """1 MiB file uploaded and read back — exercises the buffer-then-PUT+ write path under non-trivial load."""+ ctx = _provision(transfer, s3)+ payload = (b"abcdef0123" * 1024) * 100 # ~1 MiB++ async def _run():+ async with await _connect(ctx["user_name"], ctx["priv_pem"]) as conn:+ async with conn.start_sftp_client() as sftp:+ async with sftp.open("/big.bin", "wb") as f:+ await f.write(payload)+ async with sftp.open("/big.bin", "rb") as f:+ got = await f.read()+ assert got == payload++ asyncio.run(_run())+ transfer.delete_server(ServerId=ctx["server_id"])+++def test_sftp_mkdir_listdir_rmdir(transfer, s3):+ """mkdir creates a zero-byte placeholder under the prefix; listdir+ sees it as a directory; rmdir removes it (only when empty)."""+ ctx = _provision(transfer, s3)++ async def _run():+ async with await _connect(ctx["user_name"], ctx["priv_pem"]) as conn:+ async with conn.start_sftp_client() as sftp:+ await sftp.mkdir("/inbox")+ names = await sftp.listdir("/")+ assert "inbox" in names++ # rmdir on non-empty must fail; create a file then try.+ async with sftp.open("/inbox/x", "wb") as f:+ await f.write(b"x")+ with pytest.raises(asyncssh.SFTPError):+ await sftp.rmdir("/inbox")++ # Remove the file, then rmdir succeeds.+ await sftp.remove("/inbox/x")+ await sftp.rmdir("/inbox")++ asyncio.run(_run())+ transfer.delete_server(ServerId=ctx["server_id"])+++def test_sftp_logical_home_directory_mappings(transfer, s3):+ """LOGICAL home dir maps virtual `/inbox` → bucket prefix; an upload+ to /inbox/foo lands at the mapped S3 key."""+ suffix = uuid.uuid4().hex[:8]+ bucket = f"sftp-logical-{suffix}"+ s3.create_bucket(Bucket=bucket)+ ctx = _provision(+ transfer,+ s3,+ bucket=bucket,+ logical_mappings=[+ {"Entry": "/inbox", "Target": f"/{bucket}/incoming"},+ {"Entry": "/archive", "Target": f"/{bucket}/old"},+ ],+ )++ async def _run():+ async with await _connect(ctx["user_name"], ctx["priv_pem"]) as conn:+ async with conn.start_sftp_client() as sftp:+ async with sftp.open("/inbox/today.txt", "wb") as f:+ await f.write(b"hi")++ asyncio.run(_run())++ obj = s3.get_object(Bucket=bucket, Key="incoming/today.txt")+ assert obj["Body"].read() == b"hi"+ transfer.delete_server(ServerId=ctx["server_id"])+++# ---------------------------------------------------------------------------+# Multi-user / multi-server+# ---------------------------------------------------------------------------+++def test_sftp_concurrent_uploads_two_users(transfer, s3):+ """Two distinct users on two distinct servers each upload concurrently+ — exercises the SSH key disambiguation path (single-port mode)."""+ a = _provision(transfer, s3)+ b = _provision(transfer, s3)++ async def _upload(ctx, body):+ async with await _connect(ctx["user_name"], ctx["priv_pem"]) as conn:+ async with conn.start_sftp_client() as sftp:+ async with sftp.open("/concurrent.bin", "wb") as f:+ await f.write(body)++ async def _run():+ await asyncio.gather(+ _upload(a, b"AAAA"),+ _upload(b, b"BBBB"),+ )++ asyncio.run(_run())++ assert s3.get_object(Bucket=a["bucket"], Key="concurrent.bin")["Body"].read() == b"AAAA"+ assert s3.get_object(Bucket=b["bucket"], Key="concurrent.bin")["Body"].read() == b"BBBB"+ transfer.delete_server(ServerId=a["server_id"])+ transfer.delete_server(ServerId=b["server_id"])+++def test_sftp_key_disambiguates_overlapping_usernames(transfer, s3):+ """Two servers, same UserName, different keys: each user's key routes+ to the correct server's bucket. This is the AWS-faithful single-port+ behavior — username alone isn't enough, the SSH key is the+ disambiguator."""+ suffix = uuid.uuid4().hex[:8]+ user_name = f"shared-{suffix}"+ bucket_a = f"sftp-share-a-{suffix}"+ bucket_b = f"sftp-share-b-{suffix}"+ s3.create_bucket(Bucket=bucket_a)+ s3.create_bucket(Bucket=bucket_b)++ sid_a = transfer.create_server()["ServerId"]+ sid_b = transfer.create_server()["ServerId"]+ priv_a, pub_a = _gen_keypair()+ priv_b, pub_b = _gen_keypair()+ transfer.create_user(+ ServerId=sid_a, UserName=user_name,+ Role="arn:aws:iam::000000000000:role/r",+ HomeDirectoryType="PATH", HomeDirectory=f"/{bucket_a}",+ SshPublicKeyBody=pub_a,+ )+ transfer.create_user(+ ServerId=sid_b, UserName=user_name,+ Role="arn:aws:iam::000000000000:role/r",+ HomeDirectoryType="PATH", HomeDirectory=f"/{bucket_b}",+ SshPublicKeyBody=pub_b,+ )++ async def _put(priv, body):+ async with await _connect(user_name, priv) as conn:+ async with conn.start_sftp_client() as sftp:+ async with sftp.open("/marker", "wb") as f:+ await f.write(body)++ asyncio.run(_put(priv_a, b"from-a"))+ asyncio.run(_put(priv_b, b"from-b"))++ assert s3.get_object(Bucket=bucket_a, Key="marker")["Body"].read() == b"from-a"+ assert s3.get_object(Bucket=bucket_b, Key="marker")["Body"].read() == b"from-b"+ transfer.delete_server(ServerId=sid_a)+ transfer.delete_server(ServerId=sid_b)+++# ---------------------------------------------------------------------------+# Per-server-port mode (env-gated)+# ---------------------------------------------------------------------------+++def _sftp_port_state():+ """Hit /_ministack/transfer/sftp-ports — returns the JSON dict, or None+ on any failure (used to skip per-server-mode tests cleanly)."""+ import json as _json+ try:+ with urllib.request.urlopen(+ f"{ADMIN_BASE}/_ministack/transfer/sftp-ports", timeout=2+ ) as resp:+ return _json.loads(resp.read())+ except Exception:+ return None+++ not (_sftp_port_state() or {}).get("port_per_server"),+ reason="MiniStack server not started with SFTP_PORT_PER_SERVER=1",+)+def test_sftp_port_per_server_mode(transfer, s3):+ """In per-server mode, the admin endpoint surfaces a unique port for+ each Transfer server and connecting on that port reaches only that+ server's users. Mirrors AWS Transfer's per-server endpoint model."""+ ctx = _provision(transfer, s3)+ state = _sftp_port_state()+ port = state["per_server"].get(ctx["server_id"])+ assert port and port != SFTP_PORT, "expected a unique per-server SFTP port"++ async def _run():+ async with await _connect(ctx["user_name"], ctx["priv_pem"], port=port) as conn:+ async with conn.start_sftp_client() as sftp:+ async with sftp.open("/per-server-marker", "wb") as f:+ await f.write(b"ok")++ asyncio.run(_run())+ obj = s3.get_object(Bucket=ctx["bucket"], Key="per-server-marker")+ assert obj["Body"].read() == b"ok"+ transfer.delete_server(ServerId=ctx["server_id"])