The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(terminal): keep the idle reaper off an environment with a call in flight
unsupportedpastelse24c9a508699
diff --git a/tests/tools/test_terminal_inflight_reap_guard.py b/tests/tools/test_terminal_inflight_reap_guard.pynew file mode 100644index 00000000000..0b832fe99c2--- /dev/null+++ b/tests/tools/test_terminal_inflight_reap_guard.py@@ -0,0 +1,82 @@+"""The idle reaper must not tear down an environment while a foreground call is running in it.++``_cleanup_inactive_envs`` reaps any key whose ``_last_activity`` is older than ``lifetime_seconds``+(default 300 s). Only background processes refresh that stamp mid-run, while a foreground command may+run for up to 600 s: Docker workspace containers were stopped (exit 137), Modal sandboxes terminated+and SSH lost its control master mid-command.+"""++from __future__ import annotations++import json+import time++import pytest++from tools import code_execution_tool, terminal_tool as tt+from tools.terminal_tool_lifecycle import _cleanup_inactive_envs+++class _ReapingEnv:+ """An env whose command outlives ``lifetime_seconds``: one reaper sweep fires from inside it."""++ def __init__(self):+ self.cleaned = False+ self.cleaned_mid_call = None++ def cleanup(self):+ self.cleaned = True++ def execute(self, command, cwd=None, timeout=None, **kwargs):+ _sweep_after_long_call()+ self.cleaned_mid_call = self.cleaned+ return {"output": "hi", "returncode": 0}+++def _sweep_after_long_call():+ for key in list(tt._last_activity):+ tt._last_activity[key] = time.time() - 600+ _cleanup_inactive_envs(lifetime_seconds=300)+++def env(monkeypatch):+ """A shared env cached under the collapsed key ``default`` (the raw task id ``session-42`` has+ no session key or isolation override), with only the sweeps a test fires."""+ monkeypatch.setattr(tt, "_active_environments", {})+ monkeypatch.setattr(tt, "_last_activity", {})+ monkeypatch.setattr(tt, "_creation_locks", {})+ monkeypatch.setattr(tt, "_start_cleanup_thread", lambda: None)+ reaping = _ReapingEnv()+ tt._active_environments["default"] = reaping+ tt._last_activity["default"] = time.time()+ return reaping+++def test_a_foreground_terminal_call_keeps_its_environment_past_the_lifetime(env):+ result = json.loads(tt.terminal_tool(command="echo hi", task_id="session-42"))++ assert env.cleaned_mid_call is False+ assert tt._active_environments["default"] is env+ assert result["exit_code"] == 0+ # Idle again once the call returns: the countdown restarts from the call's end.+ _sweep_after_long_call()+ assert env.cleaned+++def test_a_remote_execute_code_call_keeps_its_environment_past_the_lifetime(env):+ code_execution_tool._execute_remote("print(1)", "session-42", None)++ assert env.cleaned_mid_call is False+ assert tt._active_environments["default"] is env+++def test_a_call_that_raises_releases_its_environment_to_the_reaper(env, monkeypatch):+ def boom(*args, **kwargs):+ raise RuntimeError("boom")++ monkeypatch.setattr(tt, "_run_approval_guards", boom)+ tt.terminal_tool(command="echo hi", task_id="session-42")++ _sweep_after_long_call()+ assert env.cleaneddiff --git a/tools/code_execution_tool.py b/tools/code_execution_tool.pyindex 2198c5ab711..6333d6c5442 100644--- a/tools/code_execution_tool.py+++ b/tools/code_execution_tool.py@@ -662,40 +662,42 @@ def _execute_remote(code: str, task_id: Optional[str], enabled_tools: Optional[l"""Run code on the remote terminal backend: the owner's persistent remote session kernel(tools/code_kernel_remote.py) first, else the per-call script ship — the fail-open route whena kernel cannot be spawned and the only route for hosts that cannot sustain a background process."""+ from tools.terminal_tool import _call_in_flight, _resolve_container_task_id_cfg = _load_config()timeout, max_tool_calls = _cfg.get("timeout", DEFAULT_TIMEOUT), _cfg.get("max_tool_calls", DEFAULT_MAX_TOOL_CALLS)sandbox_tools, effective_task_id = _sandbox_tools_for(enabled_tools), task_id or "default"env, env_type = _get_or_create_env(effective_task_id)- exec_start = time.monotonic()- try:- py_check = env.execute("command -v python3 >/dev/null 2>&1 && echo OK", cwd="/", timeout=15)- if "OK" not in py_check.get("output", ""):- return _error_result(f"Python 3 is not available in the {env_type} terminal "- "environment. Install Python to use execute_code with remote backends.")- # Session-kernel path: one persistent kernel per owner on the- # run-to-completion transport. Spawn failure falls OPEN to the per-call- # path below so a degraded remote host never blocks execution.+ with _call_in_flight(_resolve_container_task_id(effective_task_id)):+ exec_start = time.monotonic()try:- # --- Session-kernel path (hermes-agent#96873) ------------------- Same always-on model as- # local: one persistent kernel per owner, rebuilt on the run-to-completion transport (detached- # runner + file cell protocol).- from tools.code_kernel_remote import execute_in_remote_kernel- kernel_result = execute_in_remote_kernel(- code, env=env, env_type=env_type, task_env_id=effective_task_id,- sandbox_tools=frozenset(sandbox_tools), timeout=timeout,- max_tool_calls=max_tool_calls, reset=bool(reset),- idle_exit=int(_cfg.get("kernel_idle_timeout", 1800)),- )- except Exception:- logger.warning("remote session-kernel path failed; falling back to per-call", exc_info=True)- kernel_result = None- if kernel_result is not None:- return _finish_remote_kernel_result(kernel_result, timeout=timeout, exec_start=exec_start)- logger.info("remote session kernel unavailable on %s; using per-call path", env_type)- except Exception as exc:- return _remote_failure(exc, exec_start, 0)- return _run_remote_per_call(env, env_type, code, effective_task_id, sandbox_tools,- timeout=timeout, max_tool_calls=max_tool_calls, exec_start=exec_start)+ py_check = env.execute("command -v python3 >/dev/null 2>&1 && echo OK", cwd="/", timeout=15)+ if "OK" not in py_check.get("output", ""):+ return _error_result(f"Python 3 is not available in the {env_type} terminal "+ "environment. Install Python to use execute_code with remote backends.")+ # Session-kernel path: one persistent kernel per owner on the+ # run-to-completion transport. Spawn failure falls OPEN to the per-call+ # path below so a degraded remote host never blocks execution.+ try:+ # --- Session-kernel path (hermes-agent#96873) ------------------- Same always-on model as+ # local: one persistent kernel per owner, rebuilt on the run-to-completion transport (detached+ # runner + file cell protocol).+ from tools.code_kernel_remote import execute_in_remote_kernel+ kernel_result = execute_in_remote_kernel(+ code, env=env, env_type=env_type, task_env_id=effective_task_id,+ sandbox_tools=frozenset(sandbox_tools), timeout=timeout,+ max_tool_calls=max_tool_calls, reset=bool(reset),+ idle_exit=int(_cfg.get("kernel_idle_timeout", 1800)),+ )+ except Exception:+ logger.warning("remote session-kernel path failed; falling back to per-call", exc_info=True)+ kernel_result = None+ if kernel_result is not None:+ return _finish_remote_kernel_result(kernel_result, timeout=timeout, exec_start=exec_start)+ logger.info("remote session kernel unavailable on %s; using per-call path", env_type)+ except Exception as exc:+ return _remote_failure(exc, exec_start, 0)+ return _run_remote_per_call(env, env_type, code, effective_task_id, sandbox_tools,+ timeout=timeout, max_tool_calls=max_tool_calls, exec_start=exec_start)# ---- Main entry point ----diff --git a/tools/terminal_tool.py b/tools/terminal_tool.pyindex c079382321e..f6b75a6398b 100644--- a/tools/terminal_tool.py+++ b/tools/terminal_tool.py@@ -26,6 +26,7 @@ import sysimport timeimport threadingimport atexit+from contextlib import contextmanagerfrom dataclasses import dataclassfrom typing import Optional, Dict, Any, List@@ -185,6 +186,7 @@ _last_activity: dict[str, float] = {}_env_lock = threading.Lock()_creation_locks: dict[str, threading.Lock] = {} # Per-task locks for sandbox creation_creation_locks_lock = threading.Lock() # Protects _creation_locks dict itself+_calls_in_flight: dict[str, int] = {} # Foreground calls running per env key; the reaper skips these_cleanup_thread = None_cleanup_running = False@@ -1299,6 +1301,31 @@ def _acquire_env(plan: _ExecPlan, task_id: Optional[str]) -> Any:return new_env+@contextmanager+def _call_in_flight(*keys: Optional[str]):+ """Hold the env cached under *keys* against the idle reaper for the duration of a call.++ Only background processes refresh ``_last_activity`` mid-run, so a foreground command+ longer than ``lifetime_seconds`` had its sandbox reaped out from under it. Refcounted+ (parallel calls share a key); exit restamps, so the idle countdown starts at call end.+ """+ held = [k for k in dict.fromkeys(keys) if k]+ with _env_lock:+ for k in held:+ _calls_in_flight[k] = _calls_in_flight.get(k, 0) + 1+ try:+ yield+ finally:+ with _env_lock:+ now = time.time()+ for k in held:+ left = _calls_in_flight.pop(k) - 1+ if left:+ _calls_in_flight[k] = left+ if k in _last_activity:+ _last_activity[k] = now++def _yield_kwargs(command: str, **ctx) -> dict:"""``env.execute`` kwargs enabling yield-to-background (local backend only)."""handler = yield_to_background_handler(command=command, **ctx)@@ -1483,77 +1510,79 @@ def terminal_tool(# below (lifecycle, workdir, self-repo, dangerous-command approval) sees what runs.command, background = auto_bg, Trueenv = _acquire_env(plan, task_id)- env_type, cwd, effective_task_id = plan.env_type, plan.cwd, plan.effective_task_id-- # Session key for cwd records: the contextvar doesn't cross tool-worker- # threads, so fall back to the raw task_id (the top-level agent's- # session_key) as a stable anchor.- from tools.approval import get_current_session_key-- session_key = get_current_session_key(default="") or (task_id or "")-- # The supervised-gateway identity probe ends in a kernel process query- # (psutil create_time) that has wedged for the better part of an hour on- # macOS; ``env.execute`` is already behind ``run_bounded_sync`` but this- # chain ran ahead of it, so the tool call never returned and the cron- # slot stayed occupied (#111922). Share the command's own deadline. A- # guard that never rendered a verdict fails CLOSED: these checks apply- # unconditionally (``force`` cannot bypass them), so the command is- # refused with a retryable error instead of running unguarded.- from agent.deadline import run_bounded_sync- from tools.interrupt import acting_for_tid-- # The guard chain runs on the deadline worker; keep it answerable to /stop- # aimed at this tool thread (a remote-backend script read polls is_interrupted()).- guard_timeout = max(plan.effective_timeout, _PRE_EXEC_GUARD_MIN_TIMEOUT_S)- _acting_token = acting_for_tid.set(threading.current_thread().ident)- try:- bounded_guard = run_bounded_sync(- lambda: _pre_exec_block(- command, env=env, env_type=env_type, cwd=cwd, workdir=workdir, session_key=session_key,- ),- guard_timeout,- label="terminal.pre-exec-guard",- )- finally:- acting_for_tid.reset(_acting_token)- if bounded_guard.timed_out:- raise _Rejected(_error_json(- f"Terminal pre-execution guard did not finish within {guard_timeout}s "- "(process-identity probe wedged); the command was not run. Retry the call.",- status="error",- ))- # Pre-exec security checks (floors + dangerous command detection);- # force=True means the user already confirmed.- verdict = _run_approval_guards(command, env_type, plan.config, force=force)-- pty_disabled = pty and _command_requires_pipe_stdin(command)- if plan.promoted_from_foreground_timeout is not None:- # Promotion implies notify_on_complete; watch_patterns is a background-only flag the- # caller could not have meant for a foreground call, and the two are exclusive anyway.- background, notify_on_complete, watch_patterns = True, True, None- if background:- result = spawn_background_process(- command=command, env=env, env_type=env_type, effective_task_id=effective_task_id,- task_id=task_id, session_key=session_key, workdir=workdir, cwd=cwd,- mounted_host=getattr(env, "host_cwd", None) or plan.host_cwd,- effective_pty=pty and not pty_disabled, notify_on_complete=notify_on_complete,- watch_patterns=watch_patterns, approval_note=verdict.note,- pty_disabled_reason=_PTY_DISABLED_REASON if pty_disabled else None,- completion_output_chars=_completion_output_chars,- heartbeat_seconds=heartbeat,- persist_on_release=persist_on_release,- auto_backgrounded=auto_bg is not None and plan.promoted_from_foreground_timeout is None,- )+ # _lookup_active_env stamps whichever of the two keys holds the env.+ with _call_in_flight(plan.effective_task_id, task_id):+ env_type, cwd, effective_task_id = plan.env_type, plan.cwd, plan.effective_task_id++ # Session key for cwd records: the contextvar doesn't cross tool-worker+ # threads, so fall back to the raw task_id (the top-level agent's+ # session_key) as a stable anchor.+ from tools.approval import get_current_session_key++ session_key = get_current_session_key(default="") or (task_id or "")++ # The supervised-gateway identity probe ends in a kernel process query+ # (psutil create_time) that has wedged for the better part of an hour on+ # macOS; ``env.execute`` is already behind ``run_bounded_sync`` but this+ # chain ran ahead of it, so the tool call never returned and the cron+ # slot stayed occupied (#111922). Share the command's own deadline. A+ # guard that never rendered a verdict fails CLOSED: these checks apply+ # unconditionally (``force`` cannot bypass them), so the command is+ # refused with a retryable error instead of running unguarded.+ from agent.deadline import run_bounded_sync+ from tools.interrupt import acting_for_tid++ # The guard chain runs on the deadline worker; keep it answerable to /stop+ # aimed at this tool thread (a remote-backend script read polls is_interrupted()).+ guard_timeout = max(plan.effective_timeout, _PRE_EXEC_GUARD_MIN_TIMEOUT_S)+ _acting_token = acting_for_tid.set(threading.current_thread().ident)+ try:+ bounded_guard = run_bounded_sync(+ lambda: _pre_exec_block(+ command, env=env, env_type=env_type, cwd=cwd, workdir=workdir, session_key=session_key,+ ),+ guard_timeout,+ label="terminal.pre-exec-guard",+ )+ finally:+ acting_for_tid.reset(_acting_token)+ if bounded_guard.timed_out:+ raise _Rejected(_error_json(+ f"Terminal pre-execution guard did not finish within {guard_timeout}s "+ "(process-identity probe wedged); the command was not run. Retry the call.",+ status="error",+ ))+ # Pre-exec security checks (floors + dangerous command detection);+ # force=True means the user already confirmed.+ verdict = _run_approval_guards(command, env_type, plan.config, force=force)++ pty_disabled = pty and _command_requires_pipe_stdin(command)if plan.promoted_from_foreground_timeout is not None:- result = _with_promoted_note(result, plan.promoted_from_foreground_timeout)- return _metered(None if _host_local else plan, result)- return _metered(None if _host_local else plan, _run_foreground(- command, env, plan,- task_id=task_id, session_id=session_id, session_key=session_key,- workdir=workdir, approval_note=verdict.note, clear_interrupt=verdict.approved_run,- metered=not _host_local,- ))+ # Promotion implies notify_on_complete; watch_patterns is a background-only flag the+ # caller could not have meant for a foreground call, and the two are exclusive anyway.+ background, notify_on_complete, watch_patterns = True, True, None+ if background:+ result = spawn_background_process(+ command=command, env=env, env_type=env_type, effective_task_id=effective_task_id,+ task_id=task_id, session_key=session_key, workdir=workdir, cwd=cwd,+ mounted_host=getattr(env, "host_cwd", None) or plan.host_cwd,+ effective_pty=pty and not pty_disabled, notify_on_complete=notify_on_complete,+ watch_patterns=watch_patterns, approval_note=verdict.note,+ pty_disabled_reason=_PTY_DISABLED_REASON if pty_disabled else None,+ completion_output_chars=_completion_output_chars,+ heartbeat_seconds=heartbeat,+ persist_on_release=persist_on_release,+ auto_backgrounded=auto_bg is not None and plan.promoted_from_foreground_timeout is None,+ )+ if plan.promoted_from_foreground_timeout is not None:+ result = _with_promoted_note(result, plan.promoted_from_foreground_timeout)+ return _metered(None if _host_local else plan, result)+ return _metered(None if _host_local else plan, _run_foreground(+ command, env, plan,+ task_id=task_id, session_id=session_id, session_key=session_key,+ workdir=workdir, approval_note=verdict.note, clear_interrupt=verdict.approved_run,+ metered=not _host_local,+ ))except _Rejected as r:return r.result_jsonexcept EnvironmentConnectionError as e:diff --git a/tools/terminal_tool_lifecycle.py b/tools/terminal_tool_lifecycle.pyindex adeaeeadad4..613bc846526 100644--- a/tools/terminal_tool_lifecycle.py+++ b/tools/terminal_tool_lifecycle.py@@ -107,7 +107,7 @@ def _unregister_env(task_id: str):def _cleanup_inactive_envs(lifetime_seconds: int = 300):"""Clean up environments that have been inactive for longer than lifetime_seconds."""from tools.terminal_tool import (- _active_environments, _creation_locks, _creation_locks_lock, _env_lock,+ _active_environments, _calls_in_flight, _creation_locks, _creation_locks_lock, _env_lock,_last_activity,)current_time = time.time()@@ -128,7 +128,9 @@ def _cleanup_inactive_envs(lifetime_seconds: int = 300):# Phase 1: unregister stale entries atomically under the lock; phase 2:# stop them outside it (see _unregister_env for why).with _env_lock:- stale = [t for t, last in list(_last_activity.items()) if current_time - last > lifetime_seconds]+ # A key with a foreground call in flight is busy, however old its stamp.+ stale = [t for t, last in list(_last_activity.items())+ if current_time - last > lifetime_seconds and t not in _calls_in_flight]envs_to_stop = [(t, _active_environments.pop(t, None)) for t in stale]for t in stale:_last_activity.pop(t, None)