Skip to content

The agent that grows with you

fix(terminal): keep the idle reaper off an environment with a call in flight

unsupportedpastelse24c9a508699
diff --git a/tests/tools/test_terminal_inflight_reap_guard.py b/tests/tools/test_terminal_inflight_reap_guard.py
new file mode 100644
index 00000000000..0b832fe99c2
--- /dev/null
+++ b/tests/tools/test_terminal_inflight_reap_guard.py
@@ -0,0 +1,82 @@
+"""The idle reaper must not tear down an environment while a foreground call is running in it.
+
+``_cleanup_inactive_envs`` reaps any key whose ``_last_activity`` is older than ``lifetime_seconds``
+(default 300 s). Only background processes refresh that stamp mid-run, while a foreground command may
+run for up to 600 s: Docker workspace containers were stopped (exit 137), Modal sandboxes terminated
+and SSH lost its control master mid-command.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+
+import pytest
+
+from tools import code_execution_tool, terminal_tool as tt
+from tools.terminal_tool_lifecycle import _cleanup_inactive_envs
+
+
+class _ReapingEnv:
+ """An env whose command outlives ``lifetime_seconds``: one reaper sweep fires from inside it."""
+
+ def __init__(self):
+ self.cleaned = False
+ self.cleaned_mid_call = None
+
+ def cleanup(self):
+ self.cleaned = True
+
+ def execute(self, command, cwd=None, timeout=None, **kwargs):
+ _sweep_after_long_call()
+ self.cleaned_mid_call = self.cleaned
+ return {"output": "hi", "returncode": 0}
+
+
+def _sweep_after_long_call():
+ for key in list(tt._last_activity):
+ tt._last_activity[key] = time.time() - 600
+ _cleanup_inactive_envs(lifetime_seconds=300)
+
+
+def env(monkeypatch):
+ """A shared env cached under the collapsed key ``default`` (the raw task id ``session-42`` has
+ no session key or isolation override), with only the sweeps a test fires."""
+ monkeypatch.setattr(tt, "_active_environments", {})
+ monkeypatch.setattr(tt, "_last_activity", {})
+ monkeypatch.setattr(tt, "_creation_locks", {})
+ monkeypatch.setattr(tt, "_start_cleanup_thread", lambda: None)
+ reaping = _ReapingEnv()
+ tt._active_environments["default"] = reaping
+ tt._last_activity["default"] = time.time()
+ return reaping
+
+
+def test_a_foreground_terminal_call_keeps_its_environment_past_the_lifetime(env):
+ result = json.loads(tt.terminal_tool(command="echo hi", task_id="session-42"))
+
+ assert env.cleaned_mid_call is False
+ assert tt._active_environments["default"] is env
+ assert result["exit_code"] == 0
+ # Idle again once the call returns: the countdown restarts from the call's end.
+ _sweep_after_long_call()
+ assert env.cleaned
+
+
+def test_a_remote_execute_code_call_keeps_its_environment_past_the_lifetime(env):
+ code_execution_tool._execute_remote("print(1)", "session-42", None)
+
+ assert env.cleaned_mid_call is False
+ assert tt._active_environments["default"] is env
+
+
+def test_a_call_that_raises_releases_its_environment_to_the_reaper(env, monkeypatch):
+ def boom(*args, **kwargs):
+ raise RuntimeError("boom")
+
+ monkeypatch.setattr(tt, "_run_approval_guards", boom)
+ tt.terminal_tool(command="echo hi", task_id="session-42")
+
+ _sweep_after_long_call()
+ assert env.cleaned
diff --git a/tools/code_execution_tool.py b/tools/code_execution_tool.py
index 2198c5ab711..6333d6c5442 100644
--- a/tools/code_execution_tool.py
+++ b/tools/code_execution_tool.py
@@ -662,40 +662,42 @@ def _execute_remote(code: str, task_id: Optional[str], enabled_tools: Optional[l
"""Run code on the remote terminal backend: the owner's persistent remote session kernel
(tools/code_kernel_remote.py) first, else the per-call script ship — the fail-open route when
a kernel cannot be spawned and the only route for hosts that cannot sustain a background process."""
+ from tools.terminal_tool import _call_in_flight, _resolve_container_task_id
_cfg = _load_config()
timeout, max_tool_calls = _cfg.get("timeout", DEFAULT_TIMEOUT), _cfg.get("max_tool_calls", DEFAULT_MAX_TOOL_CALLS)
sandbox_tools, effective_task_id = _sandbox_tools_for(enabled_tools), task_id or "default"
env, env_type = _get_or_create_env(effective_task_id)
- exec_start = time.monotonic()
- try:
- py_check = env.execute("command -v python3 >/dev/null 2>&1 && echo OK", cwd="/", timeout=15)
- if "OK" not in py_check.get("output", ""):
- return _error_result(f"Python 3 is not available in the {env_type} terminal "
- "environment. Install Python to use execute_code with remote backends.")
- # Session-kernel path: one persistent kernel per owner on the
- # run-to-completion transport. Spawn failure falls OPEN to the per-call
- # path below so a degraded remote host never blocks execution.
+ with _call_in_flight(_resolve_container_task_id(effective_task_id)):
+ exec_start = time.monotonic()
try:
- # --- Session-kernel path (hermes-agent#96873) ------------------- Same always-on model as
- # local: one persistent kernel per owner, rebuilt on the run-to-completion transport (detached
- # runner + file cell protocol).
- from tools.code_kernel_remote import execute_in_remote_kernel
- kernel_result = execute_in_remote_kernel(
- code, env=env, env_type=env_type, task_env_id=effective_task_id,
- sandbox_tools=frozenset(sandbox_tools), timeout=timeout,
- max_tool_calls=max_tool_calls, reset=bool(reset),
- idle_exit=int(_cfg.get("kernel_idle_timeout", 1800)),
- )
- except Exception:
- logger.warning("remote session-kernel path failed; falling back to per-call", exc_info=True)
- kernel_result = None
- if kernel_result is not None:
- return _finish_remote_kernel_result(kernel_result, timeout=timeout, exec_start=exec_start)
- logger.info("remote session kernel unavailable on %s; using per-call path", env_type)
- except Exception as exc:
- return _remote_failure(exc, exec_start, 0)
- return _run_remote_per_call(env, env_type, code, effective_task_id, sandbox_tools,
- timeout=timeout, max_tool_calls=max_tool_calls, exec_start=exec_start)
+ py_check = env.execute("command -v python3 >/dev/null 2>&1 && echo OK", cwd="/", timeout=15)
+ if "OK" not in py_check.get("output", ""):
+ return _error_result(f"Python 3 is not available in the {env_type} terminal "
+ "environment. Install Python to use execute_code with remote backends.")
+ # Session-kernel path: one persistent kernel per owner on the
+ # run-to-completion transport. Spawn failure falls OPEN to the per-call
+ # path below so a degraded remote host never blocks execution.
+ try:
+ # --- Session-kernel path (hermes-agent#96873) ------------------- Same always-on model as
+ # local: one persistent kernel per owner, rebuilt on the run-to-completion transport (detached
+ # runner + file cell protocol).
+ from tools.code_kernel_remote import execute_in_remote_kernel
+ kernel_result = execute_in_remote_kernel(
+ code, env=env, env_type=env_type, task_env_id=effective_task_id,
+ sandbox_tools=frozenset(sandbox_tools), timeout=timeout,
+ max_tool_calls=max_tool_calls, reset=bool(reset),
+ idle_exit=int(_cfg.get("kernel_idle_timeout", 1800)),
+ )
+ except Exception:
+ logger.warning("remote session-kernel path failed; falling back to per-call", exc_info=True)
+ kernel_result = None
+ if kernel_result is not None:
+ return _finish_remote_kernel_result(kernel_result, timeout=timeout, exec_start=exec_start)
+ logger.info("remote session kernel unavailable on %s; using per-call path", env_type)
+ except Exception as exc:
+ return _remote_failure(exc, exec_start, 0)
+ return _run_remote_per_call(env, env_type, code, effective_task_id, sandbox_tools,
+ timeout=timeout, max_tool_calls=max_tool_calls, exec_start=exec_start)
# ---- Main entry point ----
diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py
index c079382321e..f6b75a6398b 100644
--- a/tools/terminal_tool.py
+++ b/tools/terminal_tool.py
@@ -26,6 +26,7 @@ import sys
import time
import threading
import atexit
+from contextlib import contextmanager
from dataclasses import dataclass
from typing import Optional, Dict, Any, List
@@ -185,6 +186,7 @@ _last_activity: dict[str, float] = {}
_env_lock = threading.Lock()
_creation_locks: dict[str, threading.Lock] = {} # Per-task locks for sandbox creation
_creation_locks_lock = threading.Lock() # Protects _creation_locks dict itself
+_calls_in_flight: dict[str, int] = {} # Foreground calls running per env key; the reaper skips these
_cleanup_thread = None
_cleanup_running = False
@@ -1299,6 +1301,31 @@ def _acquire_env(plan: _ExecPlan, task_id: Optional[str]) -> Any:
return new_env
+@contextmanager
+def _call_in_flight(*keys: Optional[str]):
+ """Hold the env cached under *keys* against the idle reaper for the duration of a call.
+
+ Only background processes refresh ``_last_activity`` mid-run, so a foreground command
+ longer than ``lifetime_seconds`` had its sandbox reaped out from under it. Refcounted
+ (parallel calls share a key); exit restamps, so the idle countdown starts at call end.
+ """
+ held = [k for k in dict.fromkeys(keys) if k]
+ with _env_lock:
+ for k in held:
+ _calls_in_flight[k] = _calls_in_flight.get(k, 0) + 1
+ try:
+ yield
+ finally:
+ with _env_lock:
+ now = time.time()
+ for k in held:
+ left = _calls_in_flight.pop(k) - 1
+ if left:
+ _calls_in_flight[k] = left
+ if k in _last_activity:
+ _last_activity[k] = now
+
+
def _yield_kwargs(command: str, **ctx) -> dict:
"""``env.execute`` kwargs enabling yield-to-background (local backend only)."""
handler = yield_to_background_handler(command=command, **ctx)
@@ -1483,77 +1510,79 @@ def terminal_tool(
# below (lifecycle, workdir, self-repo, dangerous-command approval) sees what runs.
command, background = auto_bg, True
env = _acquire_env(plan, task_id)
- env_type, cwd, effective_task_id = plan.env_type, plan.cwd, plan.effective_task_id
-
- # Session key for cwd records: the contextvar doesn't cross tool-worker
- # threads, so fall back to the raw task_id (the top-level agent's
- # session_key) as a stable anchor.
- from tools.approval import get_current_session_key
-
- session_key = get_current_session_key(default="") or (task_id or "")
-
- # The supervised-gateway identity probe ends in a kernel process query
- # (psutil create_time) that has wedged for the better part of an hour on
- # macOS; ``env.execute`` is already behind ``run_bounded_sync`` but this
- # chain ran ahead of it, so the tool call never returned and the cron
- # slot stayed occupied (#111922). Share the command's own deadline. A
- # guard that never rendered a verdict fails CLOSED: these checks apply
- # unconditionally (``force`` cannot bypass them), so the command is
- # refused with a retryable error instead of running unguarded.
- from agent.deadline import run_bounded_sync
- from tools.interrupt import acting_for_tid
-
- # The guard chain runs on the deadline worker; keep it answerable to /stop
- # aimed at this tool thread (a remote-backend script read polls is_interrupted()).
- guard_timeout = max(plan.effective_timeout, _PRE_EXEC_GUARD_MIN_TIMEOUT_S)
- _acting_token = acting_for_tid.set(threading.current_thread().ident)
- try:
- bounded_guard = run_bounded_sync(
- lambda: _pre_exec_block(
- command, env=env, env_type=env_type, cwd=cwd, workdir=workdir, session_key=session_key,
- ),
- guard_timeout,
- label="terminal.pre-exec-guard",
- )
- finally:
- acting_for_tid.reset(_acting_token)
- if bounded_guard.timed_out:
- raise _Rejected(_error_json(
- f"Terminal pre-execution guard did not finish within {guard_timeout}s "
- "(process-identity probe wedged); the command was not run. Retry the call.",
- status="error",
- ))
- # Pre-exec security checks (floors + dangerous command detection);
- # force=True means the user already confirmed.
- verdict = _run_approval_guards(command, env_type, plan.config, force=force)
-
- pty_disabled = pty and _command_requires_pipe_stdin(command)
- if plan.promoted_from_foreground_timeout is not None:
- # Promotion implies notify_on_complete; watch_patterns is a background-only flag the
- # caller could not have meant for a foreground call, and the two are exclusive anyway.
- background, notify_on_complete, watch_patterns = True, True, None
- if background:
- result = spawn_background_process(
- command=command, env=env, env_type=env_type, effective_task_id=effective_task_id,
- task_id=task_id, session_key=session_key, workdir=workdir, cwd=cwd,
- mounted_host=getattr(env, "host_cwd", None) or plan.host_cwd,
- effective_pty=pty and not pty_disabled, notify_on_complete=notify_on_complete,
- watch_patterns=watch_patterns, approval_note=verdict.note,
- pty_disabled_reason=_PTY_DISABLED_REASON if pty_disabled else None,
- completion_output_chars=_completion_output_chars,
- heartbeat_seconds=heartbeat,
- persist_on_release=persist_on_release,
- auto_backgrounded=auto_bg is not None and plan.promoted_from_foreground_timeout is None,
- )
+ # _lookup_active_env stamps whichever of the two keys holds the env.
+ with _call_in_flight(plan.effective_task_id, task_id):
+ env_type, cwd, effective_task_id = plan.env_type, plan.cwd, plan.effective_task_id
+
+ # Session key for cwd records: the contextvar doesn't cross tool-worker
+ # threads, so fall back to the raw task_id (the top-level agent's
+ # session_key) as a stable anchor.
+ from tools.approval import get_current_session_key
+
+ session_key = get_current_session_key(default="") or (task_id or "")
+
+ # The supervised-gateway identity probe ends in a kernel process query
+ # (psutil create_time) that has wedged for the better part of an hour on
+ # macOS; ``env.execute`` is already behind ``run_bounded_sync`` but this
+ # chain ran ahead of it, so the tool call never returned and the cron
+ # slot stayed occupied (#111922). Share the command's own deadline. A
+ # guard that never rendered a verdict fails CLOSED: these checks apply
+ # unconditionally (``force`` cannot bypass them), so the command is
+ # refused with a retryable error instead of running unguarded.
+ from agent.deadline import run_bounded_sync
+ from tools.interrupt import acting_for_tid
+
+ # The guard chain runs on the deadline worker; keep it answerable to /stop
+ # aimed at this tool thread (a remote-backend script read polls is_interrupted()).
+ guard_timeout = max(plan.effective_timeout, _PRE_EXEC_GUARD_MIN_TIMEOUT_S)
+ _acting_token = acting_for_tid.set(threading.current_thread().ident)
+ try:
+ bounded_guard = run_bounded_sync(
+ lambda: _pre_exec_block(
+ command, env=env, env_type=env_type, cwd=cwd, workdir=workdir, session_key=session_key,
+ ),
+ guard_timeout,
+ label="terminal.pre-exec-guard",
+ )
+ finally:
+ acting_for_tid.reset(_acting_token)
+ if bounded_guard.timed_out:
+ raise _Rejected(_error_json(
+ f"Terminal pre-execution guard did not finish within {guard_timeout}s "
+ "(process-identity probe wedged); the command was not run. Retry the call.",
+ status="error",
+ ))
+ # Pre-exec security checks (floors + dangerous command detection);
+ # force=True means the user already confirmed.
+ verdict = _run_approval_guards(command, env_type, plan.config, force=force)
+
+ pty_disabled = pty and _command_requires_pipe_stdin(command)
if plan.promoted_from_foreground_timeout is not None:
- result = _with_promoted_note(result, plan.promoted_from_foreground_timeout)
- return _metered(None if _host_local else plan, result)
- return _metered(None if _host_local else plan, _run_foreground(
- command, env, plan,
- task_id=task_id, session_id=session_id, session_key=session_key,
- workdir=workdir, approval_note=verdict.note, clear_interrupt=verdict.approved_run,
- metered=not _host_local,
- ))
+ # Promotion implies notify_on_complete; watch_patterns is a background-only flag the
+ # caller could not have meant for a foreground call, and the two are exclusive anyway.
+ background, notify_on_complete, watch_patterns = True, True, None
+ if background:
+ result = spawn_background_process(
+ command=command, env=env, env_type=env_type, effective_task_id=effective_task_id,
+ task_id=task_id, session_key=session_key, workdir=workdir, cwd=cwd,
+ mounted_host=getattr(env, "host_cwd", None) or plan.host_cwd,
+ effective_pty=pty and not pty_disabled, notify_on_complete=notify_on_complete,
+ watch_patterns=watch_patterns, approval_note=verdict.note,
+ pty_disabled_reason=_PTY_DISABLED_REASON if pty_disabled else None,
+ completion_output_chars=_completion_output_chars,
+ heartbeat_seconds=heartbeat,
+ persist_on_release=persist_on_release,
+ auto_backgrounded=auto_bg is not None and plan.promoted_from_foreground_timeout is None,
+ )
+ if plan.promoted_from_foreground_timeout is not None:
+ result = _with_promoted_note(result, plan.promoted_from_foreground_timeout)
+ return _metered(None if _host_local else plan, result)
+ return _metered(None if _host_local else plan, _run_foreground(
+ command, env, plan,
+ task_id=task_id, session_id=session_id, session_key=session_key,
+ workdir=workdir, approval_note=verdict.note, clear_interrupt=verdict.approved_run,
+ metered=not _host_local,
+ ))
except _Rejected as r:
return r.result_json
except EnvironmentConnectionError as e:
diff --git a/tools/terminal_tool_lifecycle.py b/tools/terminal_tool_lifecycle.py
index adeaeeadad4..613bc846526 100644
--- a/tools/terminal_tool_lifecycle.py
+++ b/tools/terminal_tool_lifecycle.py
@@ -107,7 +107,7 @@ def _unregister_env(task_id: str):
def _cleanup_inactive_envs(lifetime_seconds: int = 300):
"""Clean up environments that have been inactive for longer than lifetime_seconds."""
from tools.terminal_tool import (
- _active_environments, _creation_locks, _creation_locks_lock, _env_lock,
+ _active_environments, _calls_in_flight, _creation_locks, _creation_locks_lock, _env_lock,
_last_activity,
)
current_time = time.time()
@@ -128,7 +128,9 @@ def _cleanup_inactive_envs(lifetime_seconds: int = 300):
# Phase 1: unregister stale entries atomically under the lock; phase 2:
# stop them outside it (see _unregister_env for why).
with _env_lock:
- stale = [t for t, last in list(_last_activity.items()) if current_time - last > lifetime_seconds]
+ # A key with a foreground call in flight is busy, however old its stamp.
+ stale = [t for t, last in list(_last_activity.items())
+ if current_time - last > lifetime_seconds and t not in _calls_in_flight]
envs_to_stop = [(t, _active_environments.pop(t, None)) for t in stale]
for t in stale:
_last_activity.pop(t, None)