Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(egress): bound exchange resolve by an end-to-end deadline
高然d20f6cf767eb
diff --git a/components/egress/mitmscripts/system.py b/components/egress/mitmscripts/system.pyindex 30b5d4ff0..135225fc8 100644--- a/components/egress/mitmscripts/system.py+++ b/components/egress/mitmscripts/system.py@@ -71,6 +71,7 @@ import jsonimport osimport reimport socket+import timefrom contextlib import suppressfrom typing import TYPE_CHECKING, Any, Awaitable, NoReturnfrom urllib.parse import quote, quote_plus, unquote@@ -1008,7 +1009,7 @@ def _validate_active_vault_etag(value: str | None) -> str:class CredentialResolveError(Exception):"""Resolve failure with a log-safe reason: transport | invalid_response- | http_<status>. Resolve content never reaches logs."""+ | deadline | http_<status>. Resolve content never reaches logs."""def _env_int_range(name: str, default: int, bounds: tuple[int, int]) -> int:@@ -2096,7 +2097,7 @@ def _apply_vault_binding(substituted_surfaces = _apply_requestheaders_substitutions(flow, binding)if _flow_rejected(flow):- return+ return Noneinjected_names: list[str] = []for header in binding.get("headers") or []:@@ -2200,14 +2201,25 @@ async def _resolve_exchange_binding("path": flow.request.path or "/",},}+ budget = _resolve_budget_seconds()try:+ deadline = time.monotonic() + budgetasync with _resolve_limiter():- decision = await asyncio.to_thread(- _resolve_exchange_http,- socket_path,- _resolve_budget_seconds(),- path,- payload,+ # The deadline covers semaphore and executor queueing too, not+ # just the socket: a queued resolve fails within the budget+ # instead of stacking timeout batches behind a slow resolver.+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ raise CredentialResolveError("deadline")+ decision = await asyncio.wait_for(+ asyncio.to_thread(+ _resolve_exchange_http,+ socket_path,+ remaining,+ path,+ payload,+ ),+ timeout=remaining,)_install_exchange_headers(flow, vault, binding, decision)except CredentialResolveError as error:@@ -2219,6 +2231,15 @@ async def _resolve_exchange_binding(f"reason={error}")return+ except asyncio.TimeoutError:+ _reject_request(+ flow, b"credential resolution unavailable\n", status_code=407+ )+ ctx.log.warn(+ f"credential proxy: exchange resolve failed binding={name} "+ "reason=deadline"+ )+ returnexcept Exception: # noqa: BLE001 - unexpected failures fail closed too_reject_request(flow, b"credential resolution unavailable\n", status_code=407diff --git a/components/egress/tests/test_system_resolve.py b/components/egress/tests/test_system_resolve.pyindex 760ffd369..1973ce193 100644--- a/components/egress/tests/test_system_resolve.py+++ b/components/egress/tests/test_system_resolve.py@@ -296,7 +296,10 @@ class ExchangeResolveTest(TestCase):self.assertEqual(self.system.DEFAULT_CREDENTIAL_PROXY_SOCKET, socket_path)- self.assertEqual(2.25, timeout)+ # End-to-end deadline: the socket timeout is the budget remaining+ # after semaphore/executor queueing, so never more than 2.25s.+ self.assertLessEqual(timeout, 2.25)+ self.assertGreater(timeout, 2.0)self.assertEqual(self.system.CREDENTIAL_RESOLVE_PATH, path)self.assertEqual({@@ -671,7 +674,41 @@ class ExchangeResolveTest(TestCase):asyncio.run(self.system.requestheaders(flow))- self.assertEqual(expected, calls[0][1])+ timeout = calls[0][1]+ self.assertLessEqual(timeout, expected)+ self.assertGreater(timeout, expected - 0.25)++ def test_queued_resolves_fail_within_deadline(self) -> None:+ # Semaphore + executor queueing share the same budget: a request+ # queued behind a slow resolve fails at its own deadline (407),+ # never one timeout-batch later.+ os.environ[self.system.CREDENTIAL_RESOLVER_TIMEOUT_ENV] = "1"+ os.environ[self.system.CREDENTIAL_RESOLVER_MAX_CONCURRENCY_ENV] = "1"+ self._install_vault()++ def behavior(payload: dict) -> dict:+ time.sleep(1.0)+ return _parsed_decision()++ self._install_transport(behavior)+ first = _Flow()+ second = _Flow()++ async def main() -> None:+ await asyncio.gather(+ self.system.requestheaders(first),+ self.system.requestheaders(second),+ )++ asyncio.run(main())++ self.assertEqual(SECRET, first.request.headers["authorization"])+ self.assertTrue(second.metadata[self.system.FLOW_REJECTION_KEY])+ self.assertEqual(407, second.response.status_code)+ self.assertNotIn("authorization", second.request.headers)+ self.assertTrue(+ any("reason=deadline" in message for message in self.log.messages)+ )# -- transport wire format ---------------------------------------------------