Skip to content

The agent that grows with you

fix(dashboard): file and system-stats routes do their blocking I/O off the event loop (salvage #120271)

Teknium44a87e6284e7
diff --git a/hermes_cli/web_routers/files.py b/hermes_cli/web_routers/files.py
index 1f015c2f21d..9927020be3b 100644
--- a/hermes_cli/web_routers/files.py
+++ b/hermes_cli/web_routers/files.py
@@ -611,7 +611,7 @@ async def upload_chat_image(payload: ChatImageUpload, profile: Optional[str] = N
@router.get("/api/files")
-async def list_managed_files(request: Request, path: Optional[str] = None):
+def list_managed_files(request: Request, path: Optional[str] = None):
policy, target, display_path = _resolve_managed_path(path, request)
if not target.exists():
raise HTTPException(status_code=404, detail="Path not found")
@@ -668,19 +668,20 @@ def _managed_file_size(target: Path, max_bytes: int) -> int:
@router.get("/api/files/read")
-async def read_managed_file(request: Request, path: str):
+def read_managed_file(request: Request, path: str):
policy, target, display_path, max_bytes, mime_type = _managed_readable_file(request, path)
size = _managed_file_size(target, max_bytes)
with _io_errors("File is not readable", "Could not read file"):
- encoded = await asyncio.to_thread(_read_base64_file, target)
- return {
+ encoded = _read_base64_file(target)
+ # Rendered here, on the worker: returned as a dict, the multi-MB data URL would be serialized on the event loop.
+ return JSONResponse({
"name": target.name,
"path": display_path,
"size": size,
"mime_type": mime_type,
"data_url": f"data:{mime_type};base64,{encoded}",
**_managed_response_meta(policy),
- }
+ })
async def _managed_file_response(
@@ -871,7 +872,7 @@ def _managed_write_result(policy, target: Path, display_path: str) -> dict:
@router.post("/api/files/upload")
-async def upload_managed_file(payload: ManagedFileUpload, request: Request):
+def upload_managed_file(payload: ManagedFileUpload, request: Request):
policy, target, display_path = _managed_write_target(payload.path, request, payload.overwrite)
data, _mime_type = _decode_data_url(payload.data_url)
with _io_errors("File is not writable", "Could not write file"):
@@ -910,8 +911,9 @@ async def stream_upload_to_path(
total += len(chunk)
if total > _MANAGED_FILE_MAX_BYTES:
raise HTTPException(status_code=413, detail=too_large)
- out.write(chunk)
- os.replace(tmp_path, target)
+ # Only the read stays on the event loop, so an abort still reaches the cleanup below.
+ await asyncio.to_thread(out.write, chunk)
+ await asyncio.to_thread(os.replace, tmp_path, target)
renamed = True
except PermissionError:
raise HTTPException(status_code=403, detail=not_writable)
@@ -946,7 +948,7 @@ async def upload_managed_file_stream(
@router.post("/api/files/mkdir")
-async def create_managed_directory(payload: ManagedDirectoryCreate, request: Request):
+def create_managed_directory(payload: ManagedDirectoryCreate, request: Request):
policy, target, display_path = _resolve_managed_path(payload.path, request, for_write=True)
if target.exists() and not target.is_dir():
raise HTTPException(status_code=409, detail="A file already exists at that path")
@@ -956,7 +958,7 @@ async def create_managed_directory(payload: ManagedDirectoryCreate, request: Req
@router.delete("/api/files")
-async def delete_managed_file(payload: ManagedFileDelete, request: Request):
+def delete_managed_file(payload: ManagedFileDelete, request: Request):
policy, target, display_path = _resolve_managed_path(payload.path, request)
if policy.locked_root is not None and target == policy.locked_root:
raise HTTPException(status_code=400, detail="Cannot delete the managed files root")
@@ -986,6 +988,21 @@ _FS_LIST_ERRNO = (
)
+def _fs_list_entries(target: Path) -> list[dict[str, Any]]:
+ entries = []
+ with os.scandir(target) as scan:
+ for entry in scan:
+ if entry.name in _FS_READDIR_HIDDEN or _is_sensitive_path(Path(entry.path)):
+ continue
+ entries.append({
+ "name": entry.name,
+ "path": str(target / entry.name),
+ "isDirectory": entry.is_dir(follow_symlinks=False),
+ })
+ entries.sort(key=lambda item: (not item["isDirectory"], item["name"].lower(), item["name"]))
+ return entries
+
+
@router.get("/api/fs/list")
async def fs_list(path: str, request: Request, profile: Optional[str] = None):
backend = await asyncio.to_thread(_fs_backend, profile)
@@ -997,18 +1014,8 @@ async def fs_list(path: str, request: Request, profile: Optional[str] = None):
target = _agent_fs_path(path, profile_home=_fs_profile_home(profile))
_hosted_fs_read_guard(target, request)
try:
- entries = []
- with os.scandir(target) as scan:
- for entry in scan:
- if entry.name in _FS_READDIR_HIDDEN or _is_sensitive_path(Path(entry.path)):
- continue
- entries.append({
- "name": entry.name,
- "path": str(target / entry.name),
- "isDirectory": entry.is_dir(follow_symlinks=False),
- })
- entries.sort(key=lambda item: (not item["isDirectory"], item["name"].lower(), item["name"]))
- return {"entries": entries}
+ # The scan stats every entry (a network share answers in seconds): off the event loop.
+ return {"entries": await asyncio.to_thread(_fs_list_entries, target)}
except OSError as exc:
for exc_type, code in _FS_LIST_ERRNO:
if isinstance(exc, exc_type):
@@ -1205,10 +1212,12 @@ async def fs_read_data_url(
target, st = _fs_regular_file(target, _fs_profile_home(profile))
if st.st_size > _FS_DATA_URL_MAX_BYTES:
raise HTTPException(status_code=413, detail="File too large")
- encoded = await asyncio.to_thread(
- lambda: base64.b64encode(_fs_read_bytes(target)).decode("ascii"),
- )
- return {"dataUrl": f"data:{_fs_mime_type(target)};base64,{encoded}"}
+ def _render() -> JSONResponse:
+ encoded = base64.b64encode(_fs_read_bytes(target)).decode("ascii")
+ return JSONResponse({"dataUrl": f"data:{_fs_mime_type(target)};base64,{encoded}"})
+
+ # Read, encode and render off the event loop: up to 16 MB inline stalls every other request (and /api/ws).
+ return await asyncio.to_thread(_render)
def _remote_download_target(backend, path: str) -> tuple[str, int]:
@@ -1380,4 +1389,4 @@ async def fs_default_cwd(profile: Optional[str] = None):
_raise_fs_backend_error(exc)
return {"cwd": cwd, "branch": branch}
cwd = await asyncio.to_thread(_fs_default_cwd, profile)
- return {"cwd": cwd, "branch": _fs_git_branch(cwd)}
+ return {"cwd": cwd, "branch": await asyncio.to_thread(_fs_git_branch, cwd)} # a git subprocess
diff --git a/hermes_cli/web_routers/status.py b/hermes_cli/web_routers/status.py
index c0f2be118dd..833da73b66a 100644
--- a/hermes_cli/web_routers/status.py
+++ b/hermes_cli/web_routers/status.py
@@ -601,7 +601,7 @@ async def get_status(profile: Optional[str] = None):
@router.get("/api/system/stats")
-async def get_system_stats():
+def get_system_stats(): # sync: FastAPI runs it on a worker (psutil.cpu_percent blocks 100 ms)
"""Host + process system stats for the System page (stdlib identity; psutil CPU/memory/
disk/uptime when available). Non-sensitive: no env values, no paths beyond hermes home."""
import platform as _platform
diff --git a/tests/hermes_cli/test_web_server_files.py b/tests/hermes_cli/test_web_server_files.py
index ca41e0f8494..51df8982792 100644
--- a/tests/hermes_cli/test_web_server_files.py
+++ b/tests/hermes_cli/test_web_server_files.py
@@ -1,7 +1,9 @@
"""Tests for the dashboard-managed file browser API."""
+import asyncio
import base64
import json
+from pathlib import Path
from types import SimpleNamespace
import pytest
@@ -863,3 +865,85 @@ def test_dangling_symlink_does_not_break_directory_listing(forced_files_client):
# Reading or downloading the broken symlink directly is still an error.
assert client.get("/api/files/read", params={"path": str(root / "dangling")}).status_code == 404
assert client.get("/api/files/download", params={"path": str(root / "dangling")}).status_code == 404
+
+
+def _on_event_loop() -> bool:
+ try:
+ asyncio.get_running_loop()
+ return True
+ except RuntimeError:
+ return False
+
+
+def _spy_render(spy):
+ # The multi-MB data URL is rendered off the loop too: returned as a dict, FastAPI serialized it on the loop.
+ from starlette.responses import JSONResponse
+
+ spy(JSONResponse, "render",
+ lambda self, content: isinstance(content, dict) and ({"data_url", "dataUrl"} & content.keys()))
+
+
+def _route_delete(client, seeded, spy):
+ spy(_rt_files.shutil, "rmtree")
+ return client.request("DELETE", "/api/files", json={"path": str(seeded.parent), "recursive": True})
+
+
+def _route_read(client, seeded, spy):
+ _spy_render(spy)
+ spy(Path, "read_bytes", lambda self, *a, **k: self == seeded)
+ return client.get("/api/files/read", params={"path": str(seeded)})
+
+
+def _route_list(endpoint):
+ def _run(client, seeded, spy):
+ spy(_rt_files.os, "scandir", lambda target, *a, **k: Path(target) == seeded.parent)
+ return client.get(endpoint, params={"path": str(seeded.parent)})
+ return _run
+
+
+def _route_data_url(client, seeded, spy):
+ _spy_render(spy)
+ spy(_rt_files, "_fs_read_bytes")
+ return client.get("/api/fs/read-data-url", params={"path": str(seeded)})
+
+
+def _route_default_cwd(client, seeded, spy):
+ spy(_rt_files, "_fs_git_branch")
+ return client.get("/api/fs/default-cwd")
+
+
+def _route_system_stats(client, seeded, spy):
+ psutil = pytest.importorskip("psutil")
+ spy(psutil, "cpu_percent")
+ return client.get("/api/system/stats")
+
+
+_OFF_LOOP_ROUTES = {
+ "delete": _route_delete, "read": _route_read, "list": _route_list("/api/files"),
+ "fs-list": _route_list("/api/fs/list"), "fs-read-data-url": _route_data_url,
+ "fs-default-cwd": _route_default_cwd, "system-stats": _route_system_stats,
+}
+
+
[email protected]("route", sorted(_OFF_LOOP_ROUTES))
+def test_blocking_route_work_never_runs_on_the_event_loop(forced_files_client, monkeypatch, route):
+ """An ``async def`` route runs ON the event loop that also serves /api/ws chat streaming and the PTY: a recursive
+ delete, a multi-MB read, a directory scan, a git subprocess or psutil's 100 ms CPU sample there froze every other
+ request for as long as it ran. The blocking call must happen on a worker thread (#120270; from #120271)."""
+ client, root = forced_files_client
+ seeded = _seed_file(client, root)
+ seen = []
+
+ def spy(owner, name, wanted=lambda *a, **k: True):
+ original = getattr(owner, name)
+
+ def wrapper(*args, **kwargs):
+ if wanted(*args, **kwargs):
+ seen.append(_on_event_loop())
+ return original(*args, **kwargs)
+
+ monkeypatch.setattr(owner, name, wrapper)
+
+ response = _OFF_LOOP_ROUTES[route](client, seeded, spy)
+ assert response.status_code == 200, response.text
+ assert seen and not any(seen), seen