Skip to content

The agent that grows with you

fix(gateway): don't stall relay on a parked profile; retry its other ingress

David Dudok de Witcac92d05ff72
diff --git a/gateway/AGENTS.md b/gateway/AGENTS.md
index d57799eb29a..9dfd12637c3 100644
--- a/gateway/AGENTS.md
+++ b/gateway/AGENTS.md
@@ -102,6 +102,8 @@ gateway under the backend, and do NOT "fix" update locks by widening the tree-ki
with one WARNING at the first seam it reaches, never keyed into `agent:main`
(`parked_route_refusal.drop_rejected_route`; a route to a *parked* profile also tells the sender
in-channel to resend, since a direct bot's update is already confirmed and never redelivered; a
+ relay frame is ACKed and refused alike (`ParkedProfileRefusal`: a withheld one stalls the
+ ack-gated drain); a
dispatch its transport has not acknowledged yet, e.g. Google Chat HTTP events, runs in
`retryable_ingress()` and answers `ParkedRouteRefused` (503) instead, releasing its dedup claim).
`_transport_owner`, `_authorization_home_for_source`, `_resolve_profile_home_for_source`, `_session_key_profile` and
diff --git a/gateway/parked_route_refusal.py b/gateway/parked_route_refusal.py
index 59d1cae9264..c7b0c3ef274 100644
--- a/gateway/parked_route_refusal.py
+++ b/gateway/parked_route_refusal.py
@@ -1,12 +1,17 @@
"""A message a direct platform bot routes to a parked profile is refused out loud, never dropped.
-A parked profile comes back on a rescan, so its traffic is temporarily unavailable. Relay frames and
-webhooks are left unacknowledged for the platform to redeliver (``webhook_ingress``). A direct bot
-(Telegram's PTB ``Updater``, Slack Socket Mode, Discord's gateway, Google Chat Pub/Sub) has already
-confirmed the update to its platform before the gateway sees it, so the platform never redelivers it
-(Discord's own missed-message backfill, when enabled, can still pick it up after recovery). Holding it
-here would be a second inbound queue outside the one durable queue, kept for a profile whose store is
-the one that cannot be trusted. So the sender is told, in-channel, to resend, and nothing is admitted.
+A parked profile comes back on a rescan, so its traffic is temporarily unavailable. Webhooks are
+answered 503 for the provider to redeliver (``webhook_ingress``). A direct bot (Telegram's PTB
+``Updater``, Slack Socket Mode, Discord's gateway, Google Chat Pub/Sub) has already confirmed the
+update to its platform before the gateway sees it, so the platform never redelivers it (Discord's own
+missed-message backfill, when enabled, can still pick it up after recovery). Holding it here would be
+a second inbound queue outside the one durable queue, kept for a profile whose store is the one that
+cannot be trusted. So the sender is told, in-channel, to resend, and nothing is admitted.
+
+A relay frame is refused the same way (:class:`ParkedProfileRefusal`), and acknowledged: the park
+lasts until a rescan or a restart, and the connector drains its per-instance buffer in order,
+ack-gated, so a withheld frame would hold back every later frame for every profile on that instance;
+a live frame has no redelivery at all. Only transient refusals (a busy store, a drain) withhold it.
A transport that answers only after an awaited dispatch (Google Chat HTTP events) has not acknowledged
the delivery yet: it runs that dispatch in :func:`retryable_ingress` and answers the named, retryable
@@ -17,6 +22,8 @@ import contextvars
import logging
from contextlib import contextmanager
+from hermes_state_runtime import RuntimeStoreError
+
logger = logging.getLogger(__name__)
@@ -28,6 +35,16 @@ class ParkedRouteRefused(Exception):
super().__init__('The routed profile is not being served right now; retry this delivery')
+class ParkedProfileRefusal(RuntimeStoreError):
+ """Producer admission refused because the delivery routes to the parked *profile*: retryable
+ (``runtime_draining``), but for as long as the park lasts, so an ordered transport acknowledges it
+ and refuses it to the sender (:func:`refuse_parked_route`) instead of redelivering it."""
+
+ def __init__(self, profile):
+ super().__init__('runtime_draining', field='profile', detail='the routed profile is parked')
+ self.profile = profile
+
+
class _RetryableIngress:
def __init__(self):
self.open, self.refused = True, False
@@ -66,6 +83,15 @@ def drop_rejected_route(adapter, source):
logger.warning(
'[%s] Dropping inbound event for %s: explicit profile route targets an unserved profile', name, chat_id)
return
+ refuse_parked_route(adapter, source, profile)
+
+
+def refuse_parked_route(adapter, source, profile, reply_to=None):
+ """Refuse an event routed to the parked *profile*, nothing admitted: answered retryably inside
+ :func:`retryable_ingress`, else told to the sender in-channel, in reply to *reply_to* (default:
+ the source's message), off the caller, which may be the transport reader a relay send waits on."""
+ name = getattr(adapter, 'name', None) or getattr(getattr(source, 'platform', None), 'value', '?')
+ chat_id = getattr(source, 'chat_id', '?')
platform = getattr(getattr(source, 'platform', None), 'value', None) or '?'
scope = _retryable_ingress.get()
if scope is not None and scope.open:
@@ -78,7 +104,8 @@ def drop_rejected_route(adapter, source):
'[%s] Refusing inbound event for %s: routed profile %r on %s is parked (reason=profile_parked); '
'nothing was admitted, a resend notice to the sender is scheduled', name, chat_id, profile, platform)
try:
- task = asyncio.get_running_loop().create_task(_send_refusal(adapter, source))
+ task = asyncio.get_running_loop().create_task(
+ _send_refusal(adapter, source, reply_to or getattr(source, 'message_id', None)))
except RuntimeError:
logger.warning('[%s] Could not refuse the parked-profile event for %s: no running loop', name, chat_id)
return
@@ -88,12 +115,14 @@ def drop_rejected_route(adapter, source):
task.add_done_callback(tasks.discard)
-async def _send_refusal(adapter, source):
- """Reply in the sender's chat and thread, as the runner's own notices do."""
+async def _send_refusal(adapter, source, reply_to):
+ """Reply in the sender's chat and thread, as the runner's own notices do. Marked interim: it is no
+ turn's answer, so it never seals another turn's open stream (relay Slack) or fills a pending
+ answer slot (LINE's postback cache)."""
+ from gateway.run import _interim_metadata
try:
- metadata = adapter.gateway_runner._thread_metadata_for_source(source)
- result = await adapter.send(source.chat_id, PARKED_ROUTE_NOTICE, reply_to=getattr(source, 'message_id', None),
- metadata=metadata)
+ metadata = _interim_metadata(adapter.gateway_runner._thread_metadata_for_source(source))
+ result = await adapter.send(source.chat_id, PARKED_ROUTE_NOTICE, reply_to=reply_to, metadata=metadata)
if getattr(result, 'success', True) is False:
# Safe fields only: the raw ``error`` can echo tokens or message content.
logger.warning('[%s] The parked-profile refusal to %s was not delivered (error_kind=%s, retryable=%s)',
diff --git a/gateway/platforms/api_server_response_admissions.py b/gateway/platforms/api_server_response_admissions.py
index 0c7a539307d..6e606f677a2 100644
--- a/gateway/platforms/api_server_response_admissions.py
+++ b/gateway/platforms/api_server_response_admissions.py
@@ -34,6 +34,9 @@ def find_admission(adapter, durable_key, request_id):
from gateway.session_contract import SessionRef
from hermes_state_runtime import get_session_admission, _row
authority = active_authority(adapter.gateway_runner)
+ if authority is None:
+ from gateway.run_runtime import refuse_parked_scope
+ refuse_parked_scope(adapter.gateway_runner) # retryable while its profile is parked
if authority is None or adapter._ensure_session_db() is not authority.db:
raise RuntimeStoreError('profile_mismatch')
store = adapter._current_response_store()
diff --git a/gateway/platforms/shared_ingress.py b/gateway/platforms/shared_ingress.py
index 5c883734d06..322552c6c19 100644
--- a/gateway/platforms/shared_ingress.py
+++ b/gateway/platforms/shared_ingress.py
@@ -126,6 +126,11 @@ async def dispatch_profile_ingress(
from aiohttp import web
from aiohttp.web_urldispatcher import MatchInfoError
candidates = shared_ingress_apps(runner, profile)
+ from gateway.run_runtime import profile_parked
+ if not candidates and profile_parked(runner, profile):
+ # Its adapters come back with it (a rescan or restart): the platform retries a 503.
+ raise web.HTTPServiceUnavailable(text="Profile temporarily unavailable; retry",
+ headers={"Retry-After": "30"})
if not profile or not candidates:
raise web.HTTPNotFound(text="Unknown or unconfigured profile")
rel_url = request.rel_url.with_path("/" + tail.lstrip("/"), keep_query=True, keep_fragment=True)
diff --git a/gateway/platforms/webhook.py b/gateway/platforms/webhook.py
index 5d0a516b1a8..3bd608535c6 100644
--- a/gateway/platforms/webhook.py
+++ b/gateway/platforms/webhook.py
@@ -659,7 +659,7 @@ class WebhookAdapter(BasePlatformAdapter):
raw_body, error_response = await self._read_authenticated_body(request, route_name, route_config)
if error_response is None: # a parked profile is retried, before any delivery id is recorded
from gateway.platforms.webhook_ingress import parked_profile_refusal
- error_response = parked_profile_refusal(self.gateway_runner, profile)
+ error_response = parked_profile_refusal(self.gateway_runner, profile, route_name, route_config)
if error_response is not None:
return error_response
# Rate limiting (after auth)
diff --git a/gateway/platforms/webhook_ingress.py b/gateway/platforms/webhook_ingress.py
index 57ac01c3c30..eb55166abde 100644
--- a/gateway/platforms/webhook_ingress.py
+++ b/gateway/platforms/webhook_ingress.py
@@ -31,10 +31,12 @@ async def producer_scope(adapter, event):
source.profile_route_rejected = True
runtime_home = home
source._authorization_profile_home = home
- # The routed profile's own ledger admits the delivery; an unserved route is refused.
+ # The routed profile's own ledger admits the delivery; an unserved route is refused. A park
+ # can keep the profile's retiring authority (a turn outlived its Stop), which admits nothing.
authority = authority_for_home(runner, runtime_home)
- if authority is None:
+ if authority is None or getattr(authority, 'retiring', False):
_refuse_parked(runner, getattr(source, 'profile', None) or profile)
+ if authority is None:
raise RuntimeStoreError('profile_mismatch')
with native_callback(runner, event, home, profile):
async with _async_profile_runtime_scope(runtime_home):
@@ -42,25 +44,59 @@ async def producer_scope(adapter, event):
def _refuse_parked(runner, profile):
- """A delivery routed to a parked profile is retried, never refused for good: a rescan serves the
- profile again, so it is ``runtime_draining`` (no relay ACK; the connector redelivers), not the
- permanent ``profile_mismatch`` of a route to a profile this gateway does not serve."""
+ """A delivery routed to a parked profile is retryable, never refused for good: a rescan serves the
+ profile again, so it is ``runtime_draining`` (a webhook answers 503; relay acknowledges and tells
+ the sender, ``parked_route_refusal``), not the permanent ``profile_mismatch`` of a route to a
+ profile this gateway does not serve."""
from gateway.run_runtime import profile_parked
if profile_parked(runner, profile):
- raise RuntimeStoreError('runtime_draining')
+ from gateway.parked_route_refusal import ParkedProfileRefusal
+ raise ParkedProfileRefusal(profile)
-def parked_profile_refusal(runner, profile):
- """A retryable 503 for a webhook to a parked ``/p/<profile>/``, else None. Answered before the
- delivery id is recorded: coalesced, ``deliver_only`` and ``cron_job`` routes act before (or
- without) an admission, so the in-memory id is their only replay guard, and a recorded id would
- answer the provider's retry as a duplicate once the profile is served again."""
- from gateway.run_runtime import profile_parked
- if not profile_parked(runner, profile):
+def parked_profile_refusal(runner, profile, route_name, route_config):
+ """A retryable 503 for a webhook whose profile is not served right now, else None: a
+ ``/p/<profile>/`` prefix that is parked or has no serving authority here, or a coalesced route
+ that ``profile_routes`` sends to a parked profile (a coalesced delivery is acknowledged before
+ admission applies that route). Answered before the delivery id is recorded: coalesced,
+ ``deliver_only`` and ``cron_job`` routes act before (or without) an admission, so the in-memory id
+ is their only replay guard, and a recorded id would answer the provider's retry as a duplicate
+ once the profile is served again."""
+ if _prefix_not_served(runner, profile):
+ error = f'Profile {profile} is not being served; retry this delivery'
+ elif _coalesced_to_parked(runner, route_name, route_config):
+ error = 'The routed profile is not being served right now; retry this delivery'
+ else:
return None
from aiohttp import web
- return web.json_response({'error': f'Profile {profile} is not being served; retry this delivery'},
- status=503, headers={'Retry-After': '30'})
+ return web.json_response({'error': error}, status=503, headers={'Retry-After': '30'})
+
+
+def _prefix_not_served(runner, profile):
+ """Parked, or on disk with no serving authority here (its own gateway still runs, or it is not
+ picked up yet): the URL prefix resolves against the profiles on disk, the registry says what is
+ served."""
+ from gateway.run_runtime import profile_parked
+ if profile_parked(runner, profile):
+ return True
+ if not profile or getattr(runner, 'session_authorities', None) is None:
+ return False
+ from gateway.session_authorities import authority_for_home
+ from hermes_cli.profiles import get_profile_dir
+ authority = authority_for_home(runner, get_profile_dir(profile))
+ return authority is None or getattr(authority, 'retiring', False) is True
+
+
+def _coalesced_to_parked(runner, route_name, route_config):
+ routes = getattr(getattr(runner, 'config', None), 'profile_routes', None)
+ if not isinstance(route_config.get('coalesce'), dict) or not routes:
+ return False
+ from gateway.config import Platform
+ from gateway.profile_routing import match_profile_route
+ from gateway.run_runtime import profile_parked
+ # The source the admission builds carries ``user_id=webhook:<route>`` (``_admit_agent_run``).
+ route = match_profile_route(routes, platform=Platform.WEBHOOK.value, user_id=f'webhook:{route_name}')
+ return route is not None and profile_parked(runner, route.profile)
async def admit_producer(adapter, event):
diff --git a/gateway/relay/inbound_admission.py b/gateway/relay/inbound_admission.py
index ba0eb88d9fd..95c71a4f154 100644
--- a/gateway/relay/inbound_admission.py
+++ b/gateway/relay/inbound_admission.py
@@ -38,11 +38,18 @@ async def _admit_after(previous, adapter, runner, event):
async def _admit(adapter, runner, event):
# A permanent refusal returns, so the transport ACKs it (a replay would be refused again: main's
- # runner dropped it too); any other failure withholds the ACK so the connector redelivers.
+ # runner dropped it too); any other failure withholds the ACK so the connector redelivers. A
+ # parked target is refused to the sender and ACKed: it stays parked until a rescan or a restart,
+ # and a withheld frame would stall the connector's ordered, ack-gated buffer drain for every
+ # profile on this instance (a live frame is never redelivered anyway).
+ from gateway.parked_route_refusal import ParkedProfileRefusal, refuse_parked_route
from gateway.platforms.webhook_ingress import admit_producer
from gateway.relay.adapter import _PERMANENT_REFUSALS
try:
receipt = await admit_producer(adapter, event)
+ except ParkedProfileRefusal as exc:
+ refuse_parked_route(adapter, event.source, exc.profile, reply_to=event.message_id)
+ return
except RuntimeStoreError as exc:
if exc.reason not in _PERMANENT_REFUSALS:
raise
diff --git a/gateway/run_profile_reconcile.py b/gateway/run_profile_reconcile.py
index aab7bb29c85..ca5583aaaca 100644
--- a/gateway/run_profile_reconcile.py
+++ b/gateway/run_profile_reconcile.py
@@ -227,8 +227,6 @@ class GatewayProfileReconcileMixin:
if name in added:
logger.info("[MULTIPLEX] Now serving profile '%s' (%s adapter(s) connected; %s)", name, connected, reason)
result["added"].append(name)
- from gateway.run_runtime import recover_served_profile_native
- await recover_served_profile_native(self, current[name])
else:
logger.info("[MULTIPLEX] Re-scanned profile '%s' after config/.env change (%s adapter(s) connected)", name, connected)
result["rescanned"].append(name)
@@ -254,6 +252,8 @@ class GatewayProfileReconcileMixin:
if isinstance(configs, dict):
configs.pop(name, None)
if added:
+ from gateway.run_runtime import recover_served_native_rows
+ await recover_served_native_rows(self)
await self._after_profiles_added([(n, current[n]) for n in added])
result["served_profiles"] = self.served_profile_names()
return result
diff --git a/gateway/run_runtime.py b/gateway/run_runtime.py
index ca8afbcf4ec..5d0cb907605 100644
--- a/gateway/run_runtime.py
+++ b/gateway/run_runtime.py
@@ -88,6 +88,18 @@ def profile_parked(runner, name):
return bool(name) and name in (descriptor.get('parked_profiles') or {})
+def refuse_parked_scope(runner):
+ """An API request under a parked profile's ``/p/<profile>/`` prefix finds no authority: refuse it
+ retryably (``runtime_draining``, a 503 with ``Retry-After``), not with the final
+ ``profile_mismatch`` of a profile this gateway does not serve."""
+ from gateway.session_authorities import served_profile_name
+ from hermes_constants import get_hermes_home
+ name = served_profile_name(get_hermes_home())
+ if profile_parked(runner, name):
+ from gateway.parked_route_refusal import ParkedProfileRefusal
+ raise ParkedProfileRefusal(name)
+
+
def parked_profile_map(runner):
descriptor = getattr(runner, 'session_runtime_descriptor', None)
if descriptor is None:
@@ -321,7 +333,7 @@ async def serve_profile_runtime(runner, name, home):
"""Hot-serve one reserved profile's runtime: build its authority, recover its durable state and
publish it in the descriptor/ticket store — the steps boot performs per secondary. Raises when
the profile's store is unusable; the caller parks it (and releases the reservation). Its
- queued native rows are recovered once its adapters connect (``recover_served_profile_native``)."""
+ queued native rows are recovered once its adapters connect (``recover_served_native_rows``)."""
from gateway.session_authorities import owner_scope
from gateway.session_hosted_service import _ensure_hosted_service, start_ready_hosted_services
from gateway.session_local_recovery import recover_local_sessions
@@ -380,20 +392,20 @@ def _recover_published_api_turns(runner, authority):
recover_authority_api_turns(api, authority)
-async def recover_served_profile_native(runner, home):
- """Boot's native-row recovery for one hot-served profile, run once its adapters connected (boot
- runs it after every adapter connects). A store that cannot answer leaves the rows queued for
- the next start and never stops the reconcile of the other profiles."""
- registry = getattr(runner, 'session_authorities', None)
- if registry is None:
+async def recover_served_native_rows(runner):
+ """Boot's native-row recovery once hot-served profiles' adapters connected (boot runs it after
+ every adapter connects), in every serving ledger: a ``bot_profile`` route runs a bot's messages
+ on another profile, so the new bots can carry rows queued in any ledger. A session already
+ draining keeps its one drain. A store that cannot answer leaves its rows queued for the next
+ start and never stops the other ledgers or the reconcile."""
+ if getattr(runner, 'session_authorities', None) is None:
return
- authority = registry.for_home(await asyncio.to_thread(Path(home).resolve))
- if authority is None or getattr(authority, 'retiring', False):
- return
- try:
- await recover_profile_native_sessions(runner, authority)
- except Exception: # health: allow BLE001 -- logged; the rows stay queued for the next start
- logger.error('Native session recovery failed for %s', authority.profile_id, exc_info=True)
+ from gateway.session_authorities import serving_authorities
+ for authority in serving_authorities(runner):
+ try:
+ await recover_profile_native_sessions(runner, authority)
+ except Exception: # health: allow BLE001 -- logged; the rows stay queued for the next start
+ logger.error('Native session recovery failed for %s', authority.profile_id, exc_info=True)
async def unserve_profile_runtime(runner, home):
diff --git a/gateway/session_api_turn.py b/gateway/session_api_turn.py
index c0bc81d480c..19b5e503231 100644
--- a/gateway/session_api_turn.py
+++ b/gateway/session_api_turn.py
@@ -159,9 +159,11 @@ def admit_api_turn(adapter, **kwargs):
async def admit_api_turn_async(adapter, steps=None, **kwargs):
"""``admit_api_turn`` (or *steps*, a generator ending in ``admit_api_steps``) with every SQLite
write off the owner loop, in admission order."""
+ from gateway.run_runtime import refuse_parked_scope
from gateway.session_authorities import active_authority
authority = active_authority(adapter.gateway_runner)
if authority is None:
+ refuse_parked_scope(adapter.gateway_runner) # retryable while its profile is parked
raise RuntimeStoreError('profile_mismatch')
return await run_steps_off_loop(authority, steps or admit_api_steps(adapter, **kwargs), orphaned=schedule_orphan)
@@ -175,8 +177,11 @@ def schedule_orphan(admitted):
def admit_api_steps(adapter, **kwargs):
# ``/p/<profile>/`` middleware scoped this request; the routed home's authority admits it.
+ from gateway.run_runtime import refuse_parked_scope
from gateway.session_authorities import active_authority
authority = active_authority(adapter.gateway_runner)
+ if authority is None:
+ refuse_parked_scope(adapter.gateway_runner) # retryable while its profile is parked
if authority is None or adapter._ensure_session_db() is not authority.db:
raise RuntimeStoreError('profile_mismatch')
sid = kwargs.get('session_id') or uuid.uuid4().hex
diff --git a/tests/gateway/test_parked_profile_ingress.py b/tests/gateway/test_parked_profile_ingress.py
index 342d9b9d76e..96616f8a9ea 100644
--- a/tests/gateway/test_parked_profile_ingress.py
+++ b/tests/gateway/test_parked_profile_ingress.py
@@ -1,8 +1,9 @@
"""Ingress for a parked secondary is never acknowledged and then dropped.
A parked profile (here: its startup Bot recovery failed) comes back through a rescan, so its
-traffic is temporarily unavailable, not an invalid route. A relay frame is not acknowledged, so the
-connector redelivers it. A webhook gets a retryable 503 before its delivery id is recorded, so the
+traffic is temporarily unavailable, not an invalid route. A relay frame is acknowledged and the
+sender told to resend: the park can last until a manual rescan, and the connector drains its buffer
+in order, ack-gated. A webhook gets a retryable 503 before its delivery id is recorded, so the
provider's retry is processed, not answered as a duplicate.
"""
import asyncio
@@ -85,9 +86,33 @@ def _close(runner, homes):
process_ownership.release(home)
+async def _rescan_retaining_beta(runner, beta, monkeypatch):
+ """A rescan whose serve of beta fails again and whose retirement misses its deadline (a turn
+ outlived its Stop): beta is parked, its retiring authority and reservation are kept."""
+ from gateway import run_runtime
+
+ async def failing(authority):
+ raise OSError('mailbox write failed again')
+
+ async def retained(authority, timeout=None):
+ authority.retiring = True
+ return False
+
+ monkeypatch.setattr(run_runtime, 'recover_bot_mailbox', failing)
+ monkeypatch.setattr(run_runtime, '_retire_profile_authority', retained)
+ result = await runner._apply_profile_changes({}, ['beta'], [], [], reason='control-socket', live={'beta': beta})
+ assert result['parked'] == ['beta'] and runner.session_authorities.for_home(beta).retiring, result
+
+
@pytest.mark.asyncio
-async def test_a_relay_frame_for_a_parked_profile_is_redelivered_not_acked(tmp_path, monkeypatch):
[email protected]('retained', [False, True])
+async def test_a_relay_frame_for_a_parked_profile_is_refused_to_its_sender_not_held(tmp_path, monkeypatch, retained):
+ """website/docs/developer-guide/relay-connector-contract.md §3.2: on reconnect the connector drains
+ its per-instance buffer in order, ack-gated. b1 routes to the parked beta, b2 to the healthy
+ default profile and is sent only once b1 is acknowledged: holding b1 for the park would hold b2.
+ Also when the park retained beta's retiring authority, which refuses every admission."""
websockets = pytest.importorskip('websockets')
+ from gateway.parked_route_refusal import PARKED_ROUTE_NOTICE
from gateway.profile_routing import ProfileRoute
from gateway.relay.adapter import RelayAdapter
from gateway.relay.descriptor import CapabilityDescriptor
@@ -96,33 +121,45 @@ async def test_a_relay_frame_for_a_parked_profile_is_redelivered_not_acked(tmp_p
runner = await _runner_with_beta_parked(
tmp_path, monkeypatch, homes,
profile_routes=[ProfileRoute(name='to-beta', platform='telegram', profile='beta', chat_id='C')])
+ if retained:
+ await _rescan_retaining_beta(runner, homes[1][1].resolve(), monkeypatch)
descriptor = CapabilityDescriptor(1, 'telegram', 'Fixture', 4096, False, True, False, 'plain', 'chars')
- outgoing, ready, adapter = [], asyncio.Queue(), None
- frame = json.dumps({'type': 'inbound', 'bufferId': 'buf-1', 'event': {
- 'text': 'please do this', 'message_id': 'm1',
- 'source': {'platform': 'telegram', 'chat_id': 'C', 'chat_type': 'group', 'user_id': 'U'}}}) + '\n'
+ acks, sent, outbound, adapter = asyncio.Queue(), [], [], None
+
+ def frame(buffer_id, chat):
+ return json.dumps({'type': 'inbound', 'bufferId': buffer_id, 'event': {
+ 'text': f'message for {chat}', 'message_id': buffer_id,
+ 'source': {'platform': 'telegram', 'chat_id': chat, 'chat_type': 'group', 'user_id': 'U'}}}) + '\n'
async def connector(ws):
await ws.recv() # hello
await ws.send(json.dumps({'type': 'descriptor', 'descriptor': json.loads(descriptor.to_json())}) + '\n')
- await ready.put(ws)
- async for line in ws:
- outgoing.append(json.loads(line))
-
- from gateway.platforms import webhook_ingress
- real_admit, attempts = webhook_ingress.admit_producer, []
- async def recorded(adapter_, event):
+ async def reader():
+ async for line in ws:
+ msg = json.loads(line)
+ if msg.get('type') == 'inbound_ack':
+ await acks.put(msg['bufferId'])
+ elif msg.get('type') == 'outbound':
+ outbound.append(msg['action'])
+ await ws.send(json.dumps({'type': 'outbound_result', 'requestId': msg['requestId'],
+ 'result': {'success': True, 'message_id': 'notice'}}) + '\n')
+ task = asyncio.create_task(reader())
try:
- receipt = await real_admit(adapter_, event)
- except Exception as exc:
- attempts.append(exc)
- raise
- attempts.append(receipt)
- return receipt
-
- monkeypatch.setattr(webhook_ingress, 'admit_producer', recorded)
- acks = lambda: [f for f in outgoing if f.get('type') == 'inbound_ack' and f.get('bufferId') == 'buf-1']
+ for buffer_id, chat in (('b1', 'C'), ('b2', 'D')): # in order, ack-gated
+ while True:
+ sent.append(buffer_id)
+ await ws.send(frame(buffer_id, chat))
+ try:
+ if await asyncio.wait_for(acks.get(), 1.0) == buffer_id:
+ break
+ except TimeoutError:
+ continue # redeliver the unacknowledged head
+ await ws.wait_closed()
+ finally:
+ task.cancel()
+
+ told = lambda: [a for a in outbound if a.get('op') == 'send' and a.get('chat_id') == 'C']
try:
async with websockets.serve(connector, '127.0.0.1', 0) as relay:
transport = WebSocketRelayTransport(
@@ -134,19 +171,11 @@ async def test_a_relay_frame_for_a_parked_profile_is_redelivered_not_acked(tmp_p
runner.adapters[Platform.RELAY] = adapter
runner._wire_adapter_handlers(adapter)
assert await adapter.connect()
- ws = await asyncio.wait_for(ready.get(), 5)
- await ws.send(frame)
- await async_wait_until(lambda: attempts, what='the parked delivery reached admission')
- from gateway.relay.adapter import _PERMANENT_REFUSALS
- refused = attempts[0]
- assert getattr(refused, 'reason', None) not in _PERMANENT_REFUSALS, (
- f'a parked profile was refused for good (the frame is acknowledged and lost): {refused!r}')
- # The profile is served again and the connector redelivers the unacknowledged frame.
- await _serve_beta_again(runner, homes[1][1].resolve())
- await ws.send(frame)
- await async_wait_until(acks, what='buf-1 acknowledged after the redelivery')
- assert len(acks()) == 1, 'the parked delivery was acknowledged too'
- assert _admissions(homes[1][1]) == 1
+ await async_wait_until(lambda: _admissions(homes[0][1]) == 1 and told(),
+ what="the healthy profile's b2 admitted and b1's sender told")
+ assert sent.count('b1') == 1, f'the parked profile\'s frame was withheld and redelivered: {sent}'
+ assert [a['content'] for a in told()] == [PARKED_ROUTE_NOTICE]
+ assert _admissions(homes[1][1]) == 0
finally:
if adapter is not None:
await adapter.disconnect()
diff --git a/tests/gateway/test_parked_profile_rescan.py b/tests/gateway/test_parked_profile_rescan.py
index c3b67ab276c..fc7355deb50 100644
--- a/tests/gateway/test_parked_profile_rescan.py
+++ b/tests/gateway/test_parked_profile_rescan.py
@@ -134,7 +134,9 @@ async def test_api_work_queued_before_the_park_is_recovered_by_the_rescan(multip
@pytest.mark.asyncio
async def test_native_recovery_runs_once_the_rescanned_adapters_connect(multiplexed, monkeypatch):
"""Boot recovers queued native rows after the adapters connect
- (``recover_gateway_native_sessions``); the rescan runs the same step for the served profile."""
+ (``recover_gateway_native_sessions``); the rescan runs the same step once the served profile's
+ bots connect, in every serving ledger: a ``bot_profile`` route runs a bot's messages on another
+ profile, so the rows those bots carry can be queued in the default profile's ledger."""
from gateway.run_runtime import initialize_gateway_runtime
from gateway.session_authority import SessionAuthority
runner, beta = multiplexed
@@ -149,4 +151,5 @@ async def test_native_recovery_runs_once_the_rescanned_adapters_connect(multiple
await initialize_gateway_runtime(runner)
await _boot_with_beta_parked(runner, beta, monkeypatch)
await _rescan(runner, beta, monkeypatch, connected)
- assert recovered == [('beta', ['beta'])], f'native recovery after the rescan: {recovered}'
+ launch = Path(runner.session_authorities.launch.profile_id).name
+ assert sorted(recovered) == [(launch, ['beta']), ('beta', ['beta'])], f'native recovery after the rescan: {recovered}'
diff --git a/tests/gateway/test_parked_profile_unserved_ingress.py b/tests/gateway/test_parked_profile_unserved_ingress.py
new file mode 100644
index 00000000000..57d06cb12ce
--- /dev/null
+++ b/tests/gateway/test_parked_profile_unserved_ingress.py
@@ -0,0 +1,117 @@
+"""Ingress the parked-profile checks miss when they read only the URL prefix and the parked map.
+
+A coalesced webhook is acknowledged before admission applies ``profile_routes``; a ``/p/<profile>/``
+prefix can name a profile on disk that this gateway does not serve for another reason; and the API
+doors and the shared listener answer a parked profile's prefix with a final 409 or 404. Each is a
+retryable 503 here, so the client's retry is processed once the profile is served again.
+"""
+import json
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+from tests.gateway.test_parked_profile_ingress import (
+ _admissions, _cancel, _close, _homes, _runner_with_beta_parked, _serve_beta_again, _webhook)
+from tests.waiting import async_wait_until
+
+
+def _request(delivery_id, body, profile):
+ raw = json.dumps(body).encode()
+ request = MagicMock()
+ request.headers = {'X-GitHub-Delivery': delivery_id}
+ request.content_length = len(raw)
+ request.match_info = {'route_name': 'events', 'profile': profile}
+ request.method = 'POST'
+
+ async def read():
+ return raw
+
+ request.read = read
+ return request
+
+
+async def test_a_coalesced_webhook_profile_routes_send_to_a_parked_profile_is_retried_not_lost(
+ tmp_path, monkeypatch):
+ """A bare ``/webhooks/events`` that ``profile_routes`` sends to beta: a 202 would never be retried
+ by the provider, and the flush is refused while beta is parked."""
+ from gateway.platforms.webhook import _INSECURE_NO_AUTH
+ from gateway.profile_routing import ProfileRoute
+ homes = _homes(tmp_path, monkeypatch)
+ beta = homes[1][1].resolve()
+ routes = [ProfileRoute(name='hooks', platform='webhook', profile='beta', user_id='webhook:events')]
+ runner = await _runner_with_beta_parked(tmp_path, monkeypatch, homes, profile_routes=routes)
+ adapter = _webhook(runner, {'events': {'secret': _INSECURE_NO_AUTH, 'prompt': 'event {id}',
+ 'coalesce': {'key': 'id', 'window_seconds': 0.1}}})
+ settled = lambda: not adapter._coalescer.pending and not adapter._background_tasks
+ try:
+ first = await adapter._handle_webhook(_request('delivery-1', {'id': 17}, ''))
+ assert first.status == 503 and first.headers.get('Retry-After'), (first.status, first.text)
+ assert 'beta' not in first.text, 'the refusal names the routed profile'
+ assert adapter._seen_deliveries == {}, 'the refused delivery id was recorded'
+ await _serve_beta_again(runner, beta)
+ retry = await adapter._handle_webhook(_request('delivery-1', {'id': 17}, ''))
+ await async_wait_until(settled, what='the retried batch dispatched')
+ assert json.loads(retry.text)['status'] == 'coalesced', retry.text
+ assert _admissions(beta) == 1, 'delivery-1 was never admitted'
+ finally:
+ _cancel(adapter)
+ _close(runner, homes)
+
+
[email protected]('mode', ['coalesce', 'deliver_only', 'cron_job'])
+async def test_a_webhook_prefix_with_no_serving_authority_is_retried(tmp_path, monkeypatch, mode):
+ """``/p/beta/`` while beta is on disk but not served and not parked (its own gateway still runs,
+ or the watcher has not picked it up): the live registry, not the parked map, decides."""
+ from gateway.platforms.webhook import _INSECURE_NO_AUTH
+ from gateway.run_runtime import unpark_profile
+ homes = _homes(tmp_path, monkeypatch)
+ runner = await _runner_with_beta_parked(tmp_path, monkeypatch, homes)
+ unpark_profile(runner, 'beta')
+ route = {'secret': _INSECURE_NO_AUTH, 'prompt': 'event {id}', 'profile': 'beta',
+ mode: {'key': 'id', 'window_seconds': 0.1} if mode == 'coalesce' else True}
+ if mode == 'deliver_only':
+ route['deliver'] = 'log'
+ adapter = _webhook(runner, {'events': route})
+ try:
+ refused = await adapter._handle_webhook(_request('delivery-1', {'id': 17}, 'beta'))
+ assert refused.status == 503 and refused.headers.get('Retry-After'), (refused.status, refused.text)
+ assert adapter._seen_deliveries == {}, 'the refused delivery id was recorded'
+ finally:
+ _cancel(adapter)
+ _close(runner, homes)
+
+
+async def test_api_and_shared_listener_requests_to_a_parked_profile_are_retryable(tmp_path, monkeypatch):
+ """``/p/beta/v1/...`` (chat completions and runs, Responses' admission lookup) and
+ ``/p/beta/<platform callback>`` answer 503 while beta is parked, not 409 or 404."""
+ from aiohttp import web
+ from gateway.platforms.api_server_authority_runs import store_error_response
+ from gateway.platforms.api_server_response_admissions import find_admission
+ from gateway.platforms.shared_ingress import dispatch_profile_ingress
+ from gateway.run import _profile_runtime_scope
+ from gateway.session_api_turn import admit_api_steps, admit_api_turn_async
+ from hermes_state_runtime import RuntimeStoreError
+ homes = _homes(tmp_path, monkeypatch)
+ runner = await _runner_with_beta_parked(tmp_path, monkeypatch, homes)
+ api = SimpleNamespace(gateway_runner=runner)
+ refusals = []
+ try:
+ with _profile_runtime_scope(homes[1][1].resolve(), hydrate_secrets=False):
+ for door in (lambda: admit_api_turn_async(api), lambda: next(admit_api_steps(api)),
+ lambda: find_admission(api, ('key',), 'request')):
+ with pytest.raises(RuntimeStoreError) as refused:
+ pending = door()
+ if hasattr(pending, '__await__'):
+ await pending
+ refusals.append(refused.value)
+ assert [exc.reason for exc in refusals] == ['runtime_draining'] * 3
+ answer = store_error_response(lambda message, **kw: {'error': {'message': message, **kw}}, refusals[0])
+ assert answer.status == 503 and answer.headers.get('Retry-After')
+ with pytest.raises(web.HTTPServiceUnavailable):
+ await dispatch_profile_ingress(runner, 'beta', 'line/webhook', MagicMock())
+ finally:
+ _close(runner, homes)
diff --git a/tests/gateway/test_parked_route_refusal_relay.py b/tests/gateway/test_parked_route_refusal_relay.py
new file mode 100644
index 00000000000..aafddb89d42
--- /dev/null
+++ b/tests/gateway/test_parked_route_refusal_relay.py
@@ -0,0 +1,49 @@
+"""A parked-profile notice sent through relay never seals another turn's open Slack stream.
+
+On relay Slack the stream is the message: ``RelayAdapter.send`` seals a chat's single open draft with
+the first send that carries neither a per-turn identity nor the interim marker. In a channel where
+one user's turn is streaming, the notice to a second user whose message routes to a parked profile
+must go out as its own message and leave that stream open.
+"""
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from gateway.config import Platform, PlatformConfig
+from gateway.parked_route_refusal import PARKED_ROUTE_NOTICE, refuse_parked_route
+from gateway.relay.adapter import RelayAdapter
+from gateway.relay.descriptor import CapabilityDescriptor
+
+
+class _Transport:
+ _identities = [('slack', 'hermes')]
+
+ def __init__(self):
+ self.frames = []
+
+ def descriptor_for_platform(self, platform):
+ return None
+
+ async def send_outbound(self, frame, platform=None):
+ self.frames.append(frame)
+ return {'success': True, 'message_id': '9.9'}
+
+
+async def test_a_relay_notice_leaves_another_turns_open_stream_open():
+ descriptor = CapabilityDescriptor(
+ contract_version=1, platform='slack', label='Slack', max_message_length=4000,
+ supports_draft_streaming=True, supports_edit=True, supports_threads=True, markdown_dialect='mrkdwn',
+ len_unit='chars', supported_ops=('send', 'edit', 'draft', 'prompt'))
+ adapter = RelayAdapter(PlatformConfig(enabled=True, extra={}), descriptor, transport=_Transport())
+ adapter.gateway_runner = SimpleNamespace(_thread_metadata_for_source=lambda source: None)
+ await adapter.send_draft('C', 7, "U1's answer, streaming")
+ streaming = dict(adapter._open_draft_by_chat)
+ assert streaming, "test setup: U1's draft did not open"
+ source = SimpleNamespace(platform=Platform.SLACK, chat_id='C', user_id='U2')
+ refuse_parked_route(adapter, source, 'beta', reply_to='200.2')
+ await asyncio.gather(*list(adapter._background_tasks))
+ assert adapter._open_draft_by_chat == streaming, "the notice sealed U1's stream"
+ notices = [frame for frame in adapter._transport.frames if frame.get('content') == PARKED_ROUTE_NOTICE]
+ assert [frame['op'] for frame in notices] == ['send'], adapter._transport.frames