Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(egress): bound exchange resolve by an end-to-end deadline

高然d20f6cf767eb
diff --git a/components/egress/mitmscripts/system.py b/components/egress/mitmscripts/system.py
index 30b5d4ff0..135225fc8 100644
--- a/components/egress/mitmscripts/system.py
+++ b/components/egress/mitmscripts/system.py
@@ -71,6 +71,7 @@ import json
import os
import re
import socket
+import time
from contextlib import suppress
from typing import TYPE_CHECKING, Any, Awaitable, NoReturn
from urllib.parse import quote, quote_plus, unquote
@@ -1008,7 +1009,7 @@ def _validate_active_vault_etag(value: str | None) -> str:
class CredentialResolveError(Exception):
"""Resolve failure with a log-safe reason: transport | invalid_response
- | http_<status>. Resolve content never reaches logs."""
+ | deadline | http_<status>. Resolve content never reaches logs."""
def _env_int_range(name: str, default: int, bounds: tuple[int, int]) -> int:
@@ -2096,7 +2097,7 @@ def _apply_vault_binding(
substituted_surfaces = _apply_requestheaders_substitutions(flow, binding)
if _flow_rejected(flow):
- return
+ return None
injected_names: list[str] = []
for header in binding.get("headers") or []:
@@ -2200,14 +2201,25 @@ async def _resolve_exchange_binding(
"path": flow.request.path or "/",
},
}
+ budget = _resolve_budget_seconds()
try:
+ deadline = time.monotonic() + budget
async with _resolve_limiter():
- decision = await asyncio.to_thread(
- _resolve_exchange_http,
- socket_path,
- _resolve_budget_seconds(),
- path,
- payload,
+ # The deadline covers semaphore and executor queueing too, not
+ # just the socket: a queued resolve fails within the budget
+ # instead of stacking timeout batches behind a slow resolver.
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise CredentialResolveError("deadline")
+ decision = await asyncio.wait_for(
+ asyncio.to_thread(
+ _resolve_exchange_http,
+ socket_path,
+ remaining,
+ path,
+ payload,
+ ),
+ timeout=remaining,
)
_install_exchange_headers(flow, vault, binding, decision)
except CredentialResolveError as error:
@@ -2219,6 +2231,15 @@ async def _resolve_exchange_binding(
f"reason={error}"
)
return
+ except asyncio.TimeoutError:
+ _reject_request(
+ flow, b"credential resolution unavailable\n", status_code=407
+ )
+ ctx.log.warn(
+ f"credential proxy: exchange resolve failed binding={name} "
+ "reason=deadline"
+ )
+ return
except Exception: # noqa: BLE001 - unexpected failures fail closed too
_reject_request(
flow, b"credential resolution unavailable\n", status_code=407
diff --git a/components/egress/tests/test_system_resolve.py b/components/egress/tests/test_system_resolve.py
index 760ffd369..1973ce193 100644
--- a/components/egress/tests/test_system_resolve.py
+++ b/components/egress/tests/test_system_resolve.py
@@ -296,7 +296,10 @@ class ExchangeResolveTest(TestCase):
self.assertEqual(
self.system.DEFAULT_CREDENTIAL_PROXY_SOCKET, socket_path
)
- self.assertEqual(2.25, timeout)
+ # End-to-end deadline: the socket timeout is the budget remaining
+ # after semaphore/executor queueing, so never more than 2.25s.
+ self.assertLessEqual(timeout, 2.25)
+ self.assertGreater(timeout, 2.0)
self.assertEqual(self.system.CREDENTIAL_RESOLVE_PATH, path)
self.assertEqual(
{
@@ -671,7 +674,41 @@ class ExchangeResolveTest(TestCase):
asyncio.run(self.system.requestheaders(flow))
- self.assertEqual(expected, calls[0][1])
+ timeout = calls[0][1]
+ self.assertLessEqual(timeout, expected)
+ self.assertGreater(timeout, expected - 0.25)
+
+ def test_queued_resolves_fail_within_deadline(self) -> None:
+ # Semaphore + executor queueing share the same budget: a request
+ # queued behind a slow resolve fails at its own deadline (407),
+ # never one timeout-batch later.
+ os.environ[self.system.CREDENTIAL_RESOLVER_TIMEOUT_ENV] = "1"
+ os.environ[self.system.CREDENTIAL_RESOLVER_MAX_CONCURRENCY_ENV] = "1"
+ self._install_vault()
+
+ def behavior(payload: dict) -> dict:
+ time.sleep(1.0)
+ return _parsed_decision()
+
+ self._install_transport(behavior)
+ first = _Flow()
+ second = _Flow()
+
+ async def main() -> None:
+ await asyncio.gather(
+ self.system.requestheaders(first),
+ self.system.requestheaders(second),
+ )
+
+ asyncio.run(main())
+
+ self.assertEqual(SECRET, first.request.headers["authorization"])
+ self.assertTrue(second.metadata[self.system.FLOW_REJECTION_KEY])
+ self.assertEqual(407, second.response.status_code)
+ self.assertNotIn("authorization", second.request.headers)
+ self.assertTrue(
+ any("reason=deadline" in message for message in self.log.messages)
+ )
# -- transport wire format ---------------------------------------------------