Skip to content

Self-improving AI Employees for everyone

fix(resources): serialize shared browser and file operations

trohitgaea7299941e8
diff --git a/server/nodes/browser/_service.py b/server/nodes/browser/_service.py
index 092aac6c..73abae12 100644
--- a/server/nodes/browser/_service.py
+++ b/server/nodes/browser/_service.py
@@ -23,8 +23,11 @@ Windows .CMD files natively via ``CreateProcessW``, avoiding the BatBadBut
import asyncio
import json
import os
+import queue
import subprocess
-from typing import Any, Dict, List, Optional
+import threading
+from typing import Any, Callable, Dict, List, Optional
+from weakref import WeakValueDictionary
from core.logging import get_logger
from nodes.browser._install import agent_browser_binary_path
@@ -35,6 +38,38 @@ logger = get_logger(__name__)
_MAX_OUTPUT = 100_000
+async def _to_thread_until_complete(func: Callable[..., Any], /, *args: Any, **kwargs: Any) -> Any:
+ """Run sync work without abandoning it when the waiter is cancelled.
+
+ ``asyncio.to_thread`` cannot stop its worker thread. If the awaiting task
+ is cancelled while it owns a resource lock, releasing that lock would let
+ another command overlap the still-running subprocess. Defer cancellation
+ until the worker has finished (the subprocess itself has a hard timeout).
+ """
+ worker = asyncio.create_task(asyncio.to_thread(func, *args, **kwargs))
+ cancellation: Optional[asyncio.CancelledError] = None
+
+ while not worker.done():
+ try:
+ await asyncio.shield(worker)
+ except asyncio.CancelledError as exc:
+ cancellation = cancellation or exc
+ except BaseException: # The worker failed; retrieve it below.
+ break
+
+ if cancellation is not None:
+ # Retrieve a worker failure so asyncio does not report it as an
+ # unhandled task exception; the caller's cancellation remains primary.
+ if not worker.cancelled():
+ try:
+ worker.result()
+ except BaseException:
+ pass
+ raise cancellation
+
+ return worker.result()
+
+
def _max_instances() -> int:
"""Concurrent browser-session cap. Canonical value lives in
.env.template (BROWSER_MAX_INSTANCES); Settings read is wrapped
@@ -103,6 +138,10 @@ class BrowserService:
# on every command. The daemon registry stays the authority.
self._gated_sessions: set = set()
self._gate_lock = asyncio.Lock()
+ # Commands targeting one browser session share mutable tabs, refs and
+ # navigation state. Serialize only that session; unrelated sessions
+ # continue to execute in parallel.
+ self._session_locks: WeakValueDictionary[str, asyncio.Lock] = WeakValueDictionary()
async def run(
self,
@@ -117,6 +156,7 @@ class BrowserService:
auto_connect: bool = False,
chrome_profile: Optional[str] = None,
new_window: bool = False,
+ action_delay: float = 0,
) -> Dict[str, Any]:
"""Execute an agent-browser command and return parsed JSON output.
@@ -124,6 +164,55 @@ class BrowserService:
daemon process alive. We read just the first line via Popen in a
thread, then kill the process — never wait for exit.
"""
+ session_lock = self._session_locks.get(session)
+ if session_lock is None:
+ session_lock = asyncio.Lock()
+ self._session_locks[session] = session_lock
+ async with session_lock:
+ if action_delay > 0:
+ await self._run_locked(
+ args=["wait", str(action_delay)],
+ session=session,
+ timeout=timeout,
+ stdin=None,
+ headed=headed,
+ user_agent=user_agent,
+ proxy=proxy,
+ executable_path=executable_path,
+ auto_connect=auto_connect,
+ chrome_profile=chrome_profile,
+ new_window=new_window,
+ )
+ return await self._run_locked(
+ args=args,
+ session=session,
+ timeout=timeout,
+ stdin=stdin,
+ headed=headed,
+ user_agent=user_agent,
+ proxy=proxy,
+ executable_path=executable_path,
+ auto_connect=auto_connect,
+ chrome_profile=chrome_profile,
+ new_window=new_window,
+ )
+
+ async def _run_locked(
+ self,
+ *,
+ args: List[str],
+ session: str,
+ timeout: int,
+ stdin: Optional[bytes],
+ headed: bool,
+ user_agent: Optional[str],
+ proxy: Optional[str],
+ executable_path: Optional[str],
+ auto_connect: bool,
+ chrome_profile: Optional[str],
+ new_window: bool,
+ ) -> Dict[str, Any]:
+ """Execute one command while the caller holds ``session``'s lock."""
await self._enforce_instance_cap(session)
argv = [
@@ -150,7 +239,7 @@ class BrowserService:
logger.debug("agent-browser exec", argv=argv)
- raw = await asyncio.to_thread(self._run_sync, argv, timeout, stdin)
+ raw = await _to_thread_until_complete(self._run_sync, argv, timeout, stdin)
if len(raw) > _MAX_OUTPUT:
raw = raw[:_MAX_OUTPUT] + "\n...(truncated)"
@@ -177,10 +266,29 @@ class BrowserService:
names = [str(s) for s in (listing.get("data") or {}).get("sessions") or []]
if session not in names:
cap = _max_instances()
- for stale in names[: max(0, len(names) - cap + 1)]:
+ required = max(0, len(names) - cap + 1)
+ closed = 0
+ for stale in names:
+ if closed >= required:
+ break
+ stale_lock = self._session_locks.get(stale)
+ if stale_lock is not None and stale_lock.locked():
+ logger.info(
+ "[Browser] instance cap reached; preserving active session %s",
+ stale,
+ )
+ continue
logger.info("[Browser] instance cap %d reached; closing session %s", cap, stale)
await self._session_cmd(["close", "--session", stale])
self._gated_sessions.discard(stale)
+ closed += 1
+ if closed < required:
+ logger.warning(
+ "[Browser] instance cap %d temporarily exceeded; "
+ "%d session(s) are active",
+ cap,
+ required - closed,
+ )
self._gated_sessions.add(session)
async def _session_cmd(self, args: List[str]) -> Dict[str, Any]:
@@ -189,7 +297,12 @@ class BrowserService:
block an actual browser operation.
"""
try:
- raw = await asyncio.to_thread(self._run_sync, [*self._prefix, *args, "--json"], 15, None)
+ raw = await _to_thread_until_complete(
+ self._run_sync,
+ [*self._prefix, *args, "--json"],
+ 15,
+ None,
+ )
return json.loads(raw)
except Exception as e: # noqa: BLE001
logger.debug("agent-browser session command failed: args=%s error=%s", args, e)
@@ -222,6 +335,23 @@ class BrowserService:
env=_spawn_env(),
)
+ first_line: queue.Queue[tuple[bool, Any]] = queue.Queue(maxsize=1)
+
+ def _read_first_line() -> None:
+ try:
+ if proc.stdout is None:
+ raise RuntimeError("agent-browser stdout pipe was not created")
+ first_line.put((True, proc.stdout.readline()))
+ except BaseException as exc: # Hand the reader failure to caller thread.
+ first_line.put((False, exc))
+
+ reader = threading.Thread(
+ target=_read_first_line,
+ name=f"agent-browser-read-{proc.pid}",
+ daemon=True,
+ )
+ reader.start()
+
try:
if stdin_data and proc.stdin:
proc.stdin.write(stdin_data)
@@ -230,11 +360,23 @@ class BrowserService:
# Read the first line — that's the JSON result.
# The daemon keeps the process alive after this, so we must
# not call communicate() or wait() before killing it.
- line = proc.stdout.readline().decode(errors="replace").strip()
+ try:
+ ok, value = first_line.get(timeout=max(float(timeout), 0.001))
+ except queue.Empty as exc:
+ raise TimeoutError(f"agent-browser timed out after {timeout}s") from exc
+
+ if not ok:
+ raise value
+ line = value.decode(errors="replace").strip()
if not line:
# No output — check stderr for errors.
- err = proc.stderr.read().decode(errors="replace").strip()
+ # ``read`` is safe only after process exit; otherwise stderr
+ # may also be held open by the daemon and would defeat the
+ # command timeout we just enforced.
+ err = ""
+ if proc.poll() is not None and proc.stderr is not None:
+ err = proc.stderr.read().decode(errors="replace").strip()
raise RuntimeError(err or "agent-browser returned empty output")
return line
@@ -242,8 +384,26 @@ class BrowserService:
# npx -> node -> agent-browser daemon -> Chromium. Killing only
# proc.pid leaves the daemon orphaned. psutil.children(recursive=True)
# walks the tree natively on every platform.
- kill_tree(proc.pid)
- proc.wait()
+ try:
+ kill_tree(proc.pid)
+ except Exception as exc: # Ensure the direct child is still reaped.
+ logger.debug("agent-browser tree cleanup failed for pid=%s: %s", proc.pid, exc)
+ try:
+ proc.kill()
+ except OSError:
+ pass
+ try:
+ proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ try:
+ proc.kill()
+ except OSError:
+ pass
+ try:
+ proc.wait(timeout=1)
+ except subprocess.TimeoutExpired:
+ logger.warning("agent-browser process pid=%s did not exit after kill", proc.pid)
+ reader.join(timeout=1)
# -- Module-level lazy singleton (NodeJSClient pattern) -----------------
diff --git a/server/nodes/browser/browser/__init__.py b/server/nodes/browser/browser/__init__.py
index 40e82468..e57accab 100644
--- a/server/nodes/browser/browser/__init__.py
+++ b/server/nodes/browser/browser/__init__.py
@@ -378,11 +378,9 @@ class BrowserNode(ActionNode):
auto_connect=params.auto_connect,
chrome_profile=params.chrome_profile.strip() or None,
new_window=params.new_window and executable_path is not None,
+ action_delay=params.action_delay,
)
- if params.action_delay > 0:
- await svc.run(["wait", str(params.action_delay)], session, timeout, **run_kw)
-
p = params.model_dump()
if op == "batch":
cmds = json.loads(params.commands or "[]")
diff --git a/server/nodes/browser/browser_harness/_service.py b/server/nodes/browser/browser_harness/_service.py
index c9459a4e..921138dc 100644
--- a/server/nodes/browser/browser_harness/_service.py
+++ b/server/nodes/browser/browser_harness/_service.py
@@ -32,7 +32,7 @@ import asyncio
import json
import os
import subprocess
-from typing import Any, Dict, Optional
+from typing import Any, Callable, Dict, Optional
from core.logging import get_logger
from core.paths import daemons_dir
@@ -63,6 +63,35 @@ _CHROME_GUIDANCE = (
)
+async def _to_thread_until_complete(func: Callable[..., Any], /, *args: Any, **kwargs: Any) -> Any:
+ """Wait for non-cancellable thread work before propagating cancellation.
+
+ The harness subprocess owns the single daemon/CDP connection. Letting an
+ ``asyncio`` cancellation release ``_operation_lock`` while
+ ``subprocess.run`` continues would permit the next script to overlap it.
+ """
+ worker = asyncio.create_task(asyncio.to_thread(func, *args, **kwargs))
+ cancellation: Optional[asyncio.CancelledError] = None
+
+ while not worker.done():
+ try:
+ await asyncio.shield(worker)
+ except asyncio.CancelledError as exc:
+ cancellation = cancellation or exc
+ except BaseException:
+ break
+
+ if cancellation is not None:
+ if not worker.cancelled():
+ try:
+ worker.result()
+ except BaseException:
+ pass
+ raise cancellation
+
+ return worker.result()
+
+
def _runtime_dir():
d = daemons_dir() / "browser-harness"
d.mkdir(parents=True, exist_ok=True)
@@ -89,6 +118,9 @@ class BrowserHarnessService:
def __init__(self, binary: str) -> None:
self._binary = binary
+ # browser-harness owns one daemon/CDP websocket. Concurrent scripts
+ # would otherwise race over the same tabs and coordinate space.
+ self._operation_lock = asyncio.Lock()
async def run_code(self, code: str, timeout: int = 60) -> Dict[str, Any]:
"""Pipe Python ``code`` to the harness and return its output.
@@ -101,13 +133,21 @@ class BrowserHarnessService:
if not code.strip():
raise NodeUserError("code is required — Python to run against the browser-harness helpers")
- raw = await asyncio.to_thread(self._run_sync, [self._binary], code, timeout)
+ async with self._operation_lock:
+ raw = await _to_thread_until_complete(self._run_sync, [self._binary], code, timeout)
return self._shape_output(raw)
async def doctor(self) -> Dict[str, Any]:
"""Run ``browser-harness doctor``. Exit 1 = checks failed, which
is a *report*, not an error — always return the output."""
- raw = await asyncio.to_thread(self._run_sync, [self._binary, "doctor"], None, 30, check=False)
+ async with self._operation_lock:
+ raw = await _to_thread_until_complete(
+ self._run_sync,
+ [self._binary, "doctor"],
+ None,
+ 30,
+ check=False,
+ )
return {"output": raw}
@staticmethod
diff --git a/server/nodes/filesystem/_backend.py b/server/nodes/filesystem/_backend.py
index 5a0d0d5f..31cf4b6c 100644
--- a/server/nodes/filesystem/_backend.py
+++ b/server/nodes/filesystem/_backend.py
@@ -14,17 +14,94 @@ installed.
from __future__ import annotations
+import asyncio
import os
import shutil
+import stat
import subprocess
+import tempfile
from pathlib import PureWindowsPath
-from typing import Any, Dict, Optional
+from typing import Any, Callable, Dict, Optional
+from weakref import WeakValueDictionary
from deepagents.backends import LocalShellBackend
from deepagents.backends.protocol import ExecuteResponse
from deepagents.backends.utils import validate_path
+_path_locks: WeakValueDictionary[str, asyncio.Lock] = WeakValueDictionary()
+
+
+def get_path_lock(path: os.PathLike[str] | str) -> asyncio.Lock:
+ """Return the process-local lock for one resolved workspace path."""
+ key = os.path.normcase(os.path.abspath(os.fspath(path)))
+ lock = _path_locks.get(key)
+ if lock is None:
+ lock = asyncio.Lock()
+ _path_locks[key] = lock
+ return lock
+
+
+async def run_sync_until_complete(func: Callable[..., Any], /, *args: Any, **kwargs: Any) -> Any:
+ """Run sync filesystem work without abandoning it on cancellation.
+
+ Worker threads cannot be cancelled. Keep the caller suspended until the
+ mutation finishes so its path lock remains held for the whole operation,
+ then propagate the original cancellation.
+ """
+ worker = asyncio.create_task(asyncio.to_thread(func, *args, **kwargs))
+ cancellation: Optional[asyncio.CancelledError] = None
+
+ while not worker.done():
+ try:
+ await asyncio.shield(worker)
+ except asyncio.CancelledError as exc:
+ cancellation = cancellation or exc
+ except BaseException:
+ break
+
+ if cancellation is not None:
+ if not worker.cancelled():
+ try:
+ worker.result()
+ except BaseException:
+ pass
+ raise cancellation
+
+ return worker.result()
+
+
+def atomic_write_text(path: os.PathLike[str] | str, content: str) -> None:
+ """Replace ``path`` atomically with UTF-8 text from the same directory."""
+ resolved = os.path.abspath(os.fspath(path))
+ parent = os.path.dirname(resolved)
+ os.makedirs(parent, exist_ok=True)
+ existing_mode: Optional[int] = None
+ try:
+ existing_mode = stat.S_IMODE(os.stat(resolved, follow_symlinks=False).st_mode)
+ except OSError:
+ pass
+ fd, temporary = tempfile.mkstemp(prefix=".opencompany-write-", dir=parent)
+ try:
+ # newline="" preserves the caller's bytes instead of translating LF
+ # to CRLF on Windows. Preserve an existing file's permission bits
+ # before the same-directory atomic replacement where chmod is
+ # supported.
+ with os.fdopen(fd, "w", encoding="utf-8", newline="") as handle:
+ handle.write(content)
+ handle.flush()
+ os.fsync(handle.fileno())
+ if existing_mode is not None:
+ os.chmod(temporary, existing_mode)
+ os.replace(temporary, resolved)
+ except Exception:
+ try:
+ os.unlink(temporary)
+ except OSError:
+ pass
+ raise
+
+
def normalize_virtual_path(path: str) -> str:
"""Coerce any caller-supplied path to deepagents' canonical virtual form.
diff --git a/server/nodes/filesystem/file_modify/__init__.py b/server/nodes/filesystem/file_modify/__init__.py
index 0d745eca..8634131a 100644
--- a/server/nodes/filesystem/file_modify/__init__.py
+++ b/server/nodes/filesystem/file_modify/__init__.py
@@ -59,51 +59,63 @@ class FileModifyNode(ActionNode):
@Operation("modify")
async def modify(self, ctx: NodeContext, params: FileModifyParams) -> Any:
"""Inlined from handlers/filesystem.py (Wave 11.D.1)."""
- import asyncio
- from .._backend import get_backend, normalize_virtual_path
+ from deepagents.backends.utils import perform_string_replacement
+
+ from .._backend import (
+ atomic_write_text,
+ get_backend,
+ get_path_lock,
+ normalize_virtual_path,
+ run_sync_until_complete,
+ )
if not params.file_path:
raise NodeUserError("file_path is required")
backend = get_backend(params.model_dump(), ctx.raw)
file_path = normalize_virtual_path(params.file_path)
+ resolved = backend._resolve_path(file_path)
+ path_lock = get_path_lock(resolved)
if params.operation == "write":
- # ``write`` is wholesale create-or-replace. deepagents'
- # backend.write() refuses to overwrite by design, so unlink any
- # pre-existing file at the resolved path first; ``edit`` remains
- # the surgical option for callers that want a partial change.
def _do_write():
- resolved = backend._resolve_path(file_path)
- if resolved.exists():
- if resolved.is_dir():
- raise IsADirectoryError(f"Cannot write to {file_path}: path is a directory")
- resolved.unlink()
- return backend.write(file_path, params.content)
+ if resolved.exists() and resolved.is_dir():
+ raise IsADirectoryError(f"Cannot write to {file_path}: path is a directory")
+ atomic_write_text(resolved, params.content)
try:
- result = await asyncio.to_thread(_do_write)
+ async with path_lock:
+ await run_sync_until_complete(_do_write)
except (OSError, ValueError) as e:
raise NodeUserError(str(e)) from e
- if result.error:
- raise NodeUserError(result.error)
- return {"operation": "write", "file_path": result.path or file_path}
+ return {"operation": "write", "file_path": file_path}
if params.operation == "edit":
if not params.old_string:
raise NodeUserError("old_string is required for edit")
- result = await asyncio.to_thread(
- backend.edit,
- file_path,
- params.old_string,
- params.new_string,
- replace_all=params.replace_all,
- )
- if result.error:
- raise NodeUserError(result.error)
+ def _do_edit():
+ if not resolved.exists() or not resolved.is_file():
+ return None, None, f"Error: File '{file_path}' not found"
+ content = resolved.read_text(encoding="utf-8")
+ old_string = params.old_string.replace("\r\n", "\n").replace("\r", "\n")
+ new_string = params.new_string.replace("\r\n", "\n").replace("\r", "\n")
+ replacement = perform_string_replacement(content, old_string, new_string, params.replace_all)
+ if isinstance(replacement, str):
+ return None, None, replacement
+ new_content, occurrences = replacement
+ atomic_write_text(resolved, new_content)
+ return file_path, int(occurrences), None
+
+ try:
+ async with path_lock:
+ result_path, occurrences, error = await run_sync_until_complete(_do_edit)
+ except (OSError, UnicodeError, ValueError) as e:
+ raise NodeUserError(f"Error editing file '{file_path}': {e}") from e
+ if error:
+ raise NodeUserError(error)
return {
"operation": "edit",
- "file_path": result.path or file_path,
- "occurrences": result.occurrences,
+ "file_path": result_path or file_path,
+ "occurrences": occurrences,
}
raise NodeUserError(f"Unknown operation: {params.operation}")