Skip to content

The agent that grows with you

fix: safe-mode and config-only chats refuse a removed folder instead of parking their queue

Teknium256e54caac43
diff --git a/gateway/run_turn_runner.py b/gateway/run_turn_runner.py
index 8fe66327199..a70670d3d6f 100644
--- a/gateway/run_turn_runner.py
+++ b/gateway/run_turn_runner.py
@@ -171,17 +171,6 @@ def _resolution_failure_reply(exc: Exception, platform: Platform) -> tuple[str,
return t("gateway.errors.no_credentials"), auth_permanent, False
-def _workspace_unavailable_result(policy) -> dict:
- """A failed, never-run turn for a session whose frozen working directory is gone. Nothing ran
- and an unchanged resend fails the same way, so the reply names the folder and the way out
- instead of the generic "something went wrong" apology."""
- from agent.turn_failure_copy import stamp_failure
- text = t("gateway.errors.workspace_unavailable", path=policy.cwd)
- return stamp_failure({"final_response": text, "messages": [], "api_calls": 0, "tools": [],
- "failed": True, "completed": False, "error": text},
- "workspace_unavailable", False)
-
-
class TurnRunner(GatewayTurnProgressMixin, GatewaySessionAgentMixin):
"""Per-turn collaborator carrying ``GatewayRunner._run_agent_inner``'s tool-progress callbacks."""
@@ -1172,7 +1161,7 @@ class TurnRunner(GatewayTurnProgressMixin, GatewaySessionAgentMixin):
def run_sync(self):
from contextlib import ExitStack
- from gateway.session_policy import bind_turn_workspace, policy_for_source, policy_scope
+ from gateway.session_policy import bind_turn_workspace, policy_for_source, policy_scope, workspace_unavailable_result
from gateway.session_api_turn import api_policy_scope
from gateway.session_authorities import active_authority
from hermes_state_runtime import RuntimeStoreError
@@ -1183,7 +1172,7 @@ class TurnRunner(GatewayTurnProgressMixin, GatewaySessionAgentMixin):
except RuntimeStoreError as exc:
if exc.reason != 'workspace_unavailable':
raise
- result = _workspace_unavailable_result(policy)
+ result = workspace_unavailable_result(policy)
else:
scope.enter_context(api_policy_scope())
bind_turn_workspace(policy, self._ctx.session_id)
diff --git a/gateway/session_ingress.py b/gateway/session_ingress.py
index c4ce71d9636..b4b3599a26f 100644
--- a/gateway/session_ingress.py
+++ b/gateway/session_ingress.py
@@ -234,10 +234,10 @@ async def execute_admission(authority, ref, row):
from gateway.session_cron import execute
return await execute(authority, ref, row, policy)
local_policy = policy
- from gateway.session_managed_worker import managed_policy, execute_managed
+ from gateway.session_managed_worker import managed_policy, admit_managed
policy = managed_policy(authority, ref)
if policy is not None:
- return await execute_managed(authority, ref, row, policy)
+ return await admit_managed(authority, ref, row, policy)
live = authority.sessions[ref.session_id]
native = row['admission_id'] in authority.native_waiters
authority.native_waiters.discard(row['admission_id'])
diff --git a/gateway/session_managed_worker.py b/gateway/session_managed_worker.py
index b2fdfd3f2ba..86baeb5c763 100644
--- a/gateway/session_managed_worker.py
+++ b/gateway/session_managed_worker.py
@@ -579,6 +579,18 @@ async def _spawn_registered_worker(authority, ref, row):
return worker, process
+async def admit_managed(authority, ref, row, policy):
+ """The ingress route into a managed worker. A removed folder is refused before a worker exists,
+ like the in-process runner: a worker failing in its policy scope reads as a LOST execution
+ (``unknown``) and parks every follower behind it."""
+ from gateway.session_policy import workspace_ready, workspace_unavailable_result
+ if not workspace_ready(policy):
+ result = workspace_unavailable_result(policy)
+ authority.pending_results[row['admission_id']] = {'result': result, 'usage': {}}
+ return result['final_response']
+ return await execute_managed(authority, ref, row, policy)
+
+
async def execute_managed(authority, ref, row, policy):
from gateway.run_turn_progress import publish_worker_tool_event
# This route never passes the inbound prepare step, so the worker would read ``@`` tokens as typed.
diff --git a/gateway/session_policy.py b/gateway/session_policy.py
index 66778913d7e..ab3aace258b 100644
--- a/gateway/session_policy.py
+++ b/gateway/session_policy.py
@@ -1044,14 +1044,30 @@ def policy_for_source(runner, source):
return None
+def workspace_ready(policy) -> bool:
+ """Readiness, not structure: a removed workspace keeps the chat readable and movable, but no
+ turn runs in a missing directory."""
+ return Path(policy.cwd).is_dir()
+
+
+def workspace_unavailable_result(policy) -> dict:
+ """A failed, never-run turn for a session whose frozen working directory is gone. Nothing ran
+ and an unchanged resend fails the same way, so the reply names the folder and the way out
+ instead of the generic "something went wrong" apology."""
+ from agent.i18n import t
+ from agent.turn_failure_copy import stamp_failure
+ text = t("gateway.errors.workspace_unavailable", path=policy.cwd)
+ return stamp_failure({"final_response": text, "messages": [], "api_calls": 0, "tools": [],
+ "failed": True, "completed": False, "error": text},
+ "workspace_unavailable", False)
+
+
@contextmanager
def policy_scope(policy, *, authority=None):
if policy is None:
yield
return
- # Readiness, not structure: a removed workspace keeps the chat readable and movable, but no
- # turn runs in a missing directory.
- if not Path(policy.cwd).is_dir():
+ if not workspace_ready(policy):
raise RuntimeStoreError('workspace_unavailable', field='cwd', detail=policy.cwd)
from agent.runtime_cwd import set_session_cwd
from tools.terminal_scope import set_terminal_scope, reset_terminal_scope
diff --git a/tests/gateway/test_session_policy_workspace_readiness.py b/tests/gateway/test_session_policy_workspace_readiness.py
index af3df28a8a8..9435da42c2e 100644
--- a/tests/gateway/test_session_policy_workspace_readiness.py
+++ b/tests/gateway/test_session_policy_workspace_readiness.py
@@ -61,3 +61,71 @@ async def test_turn_in_removed_workspace_names_the_folder_instead_of_failing_gen
finally:
await owner.close()
store.close_all_db_handles()
+
+
+def test_managed_turn_in_removed_workspace_settles_failed_and_never_parks_the_queue(tmp_path):
+ """Safe-mode and config-only sessions run their turns in a managed worker. A removed folder used
+ to crash that worker in its policy scope, which the owner records as a LOST execution: the turn
+ settled ``unknown`` and every later message stayed queued behind it even after the folder came
+ back. Real gateway, real worker, loopback model."""
+ import asyncio
+ import json
+ import sqlite3
+ import threading
+ from http.server import ThreadingHTTPServer
+ from pathlib import Path
+
+ from tests.gateway.fixtures.local_recovery_probe import Model, child_env, daemon, rpc, websocket
+ from tests.waiting import async_wait_until
+
+ root = Path(__file__).resolve().parents[2]
+ home, user, folder = tmp_path / 'state', tmp_path / 'user', tmp_path / 'project'
+ home.mkdir(mode=0o700)
+ user.mkdir()
+ folder.mkdir()
+ peer = ThreadingHTTPServer(('127.0.0.1', 0), Model)
+ peer.requests, peer.blocked, peer.release = [], threading.Event(), threading.Event()
+ threading.Thread(target=peer.serve_forever, daemon=True).start()
+ url = f'http://127.0.0.1:{peer.server_port}/v1'
+ (home / 'config.yaml').write_text(json.dumps({
+ 'gateway': {'multiplex_profiles': False}, 'model': {'provider': 'custom', 'default': 'm', 'base_url': url},
+ 'auxiliary': {'title_generation': {'enabled': False}}, 'platform_toolsets': {'cli': []}}))
+ env = child_env()
+ env.update(HOME=str(user), USERPROFILE=str(user), HERMES_HOME=str(home), PYTHONPATH=str(root),
+ OPENAI_API_KEY='loopback-only', OPENAI_BASE_URL=url, PYTHONUNBUFFERED='1')
+
+ def admission(name):
+ with sqlite3.connect(f'file:{home / "state.db"}?mode=ro', uri=True) as db:
+ return db.execute('SELECT status, outcome FROM session_admissions WHERE request_id=?', (name,)).fetchone()
+
+ async def drive(desc):
+ async with websocket(home, desc) as ws:
+ created = await rpc(ws, 'session.create', request_id='safe', source='cli', cwd=str(folder),
+ toolsets=[], safe_mode=True, # a bypass launch names its own route
+ model='m', provider='custom', base_url=url, api_key='loopback-only')
+ assert 'result' in created, created
+ sid = created['result']['session_id']
+ folder.rmdir()
+ sent = await rpc(ws, 'prompt.submit', session_id=sid, input_id='gone', text='NEVER_RUN')
+ await async_wait_until(lambda: (admission('gone') or ('',))[0] in ('terminal', 'unknown'),
+ timeout=60, what='refused turn settled')
+ assert admission('gone') == ('terminal', 'failed')
+ receipt = await rpc(ws, 'prompt.receipt', session_id=sid, admission_id=sent['result']['admission_id'],
+ include_result=True)
+ result = receipt['result']['result']
+ assert result['failure_reason'] == 'workspace_unavailable' and result['api_calls'] == 0, result
+ assert str(folder) in result['final_response']
+ folder.mkdir()
+ await rpc(ws, 'prompt.submit', session_id=sid, input_id='back', text='FOLDER_BACK')
+ await async_wait_until(lambda: (admission('back') or ('',))[0] == 'terminal',
+ timeout=60, what='turn after the folder came back ran')
+ assert admission('back') == ('terminal', 'completed')
+
+ try:
+ with daemon(root, home, env, barrier=False) as (_proc, desc):
+ asyncio.run(drive(desc))
+ finally:
+ peer.shutdown()
+ sent_texts = json.dumps(peer.requests)
+ assert 'NEVER_RUN' not in sent_texts and 'FOLDER_BACK' in sent_texts