Skip to content

The agent that grows with you

fix(gateway): release abandoned native media captures

David Dudok de Wite834ff2025e4
diff --git a/gateway/platforms/webhook_ingress.py b/gateway/platforms/webhook_ingress.py
index 57ac01c3c30..73a22323362 100644
--- a/gateway/platforms/webhook_ingress.py
+++ b/gateway/platforms/webhook_ingress.py
@@ -163,7 +163,15 @@ async def _prepared(authority, event):
authorization as admission, so a refused sender raises here."""
import json
from gateway.session_envelope import prepare_native, restore_native
- payload = await prepare_native(authority.runner, event)
+ from gateway.session_ingress_media import capture_lease, release_abandoned_capture
+ with capture_lease(authority) as scope:
+ try:
+ payload = await prepare_native(authority.runner, event)
+ except BaseException:
+ if not scope.abandoned:
+ scope.abandoned = True
+ await release_abandoned_capture(scope)
+ raise
source = restore_native(payload).source
identity = json.dumps([source.profile, source.platform.value, source.chat_id,
source.thread_id, source.user_id], separators=(',', ':'))
diff --git a/gateway/session_authority.py b/gateway/session_authority.py
index 42ed3a7bc16..9a53a629dae 100644
--- a/gateway/session_authority.py
+++ b/gateway/session_authority.py
@@ -402,10 +402,18 @@ class SessionAuthority:
async def admit_native(self, event):
"""Await current connector policy, then commit before ACK or scheduling execution."""
- from gateway.session_ingress_media import capture_lease
+ from gateway.session_ingress_media import capture_lease, release_abandoned_capture
async with self._route_turn(event) as fence:
- with capture_lease(): # captured bytes are held from capture until commit or refusal
- return await self._admit_native(event, fence)
+ with capture_lease(self) as scope: # captured bytes are held until commit or refusal
+ try:
+ return await self._admit_native(event, fence)
+ except BaseException:
+ # Before the shielded write starts, no row can own these bytes. A cancelled copy
+ # arranges its own completion cleanup in prepare_native; other failures clean here.
+ if not scope.write_started and not scope.abandoned:
+ scope.abandoned = True
+ await release_abandoned_capture(scope)
+ raise
@contextlib.asynccontextmanager
async def _route_turn(self, event):
@@ -472,6 +480,9 @@ class SessionAuthority:
if rewrite is not None and rewrite.applied:
rewrite.on_commit()
return self._admitted(committed[0], event, row)
+ from gateway.session_ingress_media import _lease_scope
+ if (scope := _lease_scope.get()) is not None:
+ scope.write_started = True
native_write_started.set(True) # shielded from here: a cancelled caller may still commit
return await tracked_write(self, partial(
self._admit_native_write, principal_id='messaging:' + identity, session_id=ref.session_id,
diff --git a/gateway/session_envelope.py b/gateway/session_envelope.py
index 52ad7fc9342..0acc0e5e855 100644
--- a/gateway/session_envelope.py
+++ b/gateway/session_envelope.py
@@ -178,7 +178,32 @@ async def prepare_native(runner, event):
encoded_source = _validate_native(runner, frozen, provenance, fresh_roles)
# Copy + SHA-256 + fsyncs of up to max_inbound_media_bytes stay off the owner loop; the
# copied context carries the admission's capture lease.
- media = await asyncio.to_thread(capture_native_media, frozen.media_urls) if frozen.media_urls else None
+ media = None
+ if frozen.media_urls:
+ from gateway.session_ingress_media import _lease_scope, release_abandoned_capture
+ from gateway.session_runtime_workers import track_mutation
+ scope = _lease_scope.get()
+ work = asyncio.to_thread(capture_native_media, frozen.media_urls)
+ capture = (track_mutation(scope.authority, work) if scope is not None and scope.authority is not None
+ else asyncio.create_task(work))
+ try:
+ # Cancelling this waiter must not cancel the thread's completion observer: the thread
+ # may publish a path after the caller's lease context has already exited.
+ media = await asyncio.shield(capture)
+ except asyncio.CancelledError:
+ if scope is not None and scope.authority is not None:
+ scope.abandoned = True
+ def finished(done):
+ if not done.cancelled():
+ done.exception() # observe a failed copy too; partial publishes still collect
+ track_mutation(scope.authority, release_abandoned_capture(scope))
+ capture.add_done_callback(finished)
+ raise
+ except BaseException:
+ if scope is not None and scope.authority is not None:
+ scope.abandoned = True
+ await release_abandoned_capture(scope)
+ raise
return _snapshot_native(runner, frozen, provenance, fresh_roles, encoded_source, media)
diff --git a/gateway/session_ingress_media.py b/gateway/session_ingress_media.py
index c6c5092c6ad..3af05751163 100644
--- a/gateway/session_ingress_media.py
+++ b/gateway/session_ingress_media.py
@@ -37,23 +37,37 @@ _lease_lock = threading.Lock()
_lease_scope = ContextVar('native_media_lease_scope', default=None)
+class _CaptureScope(list):
+ def __init__(self, authority):
+ super().__init__()
+ self.authority = authority
+ self.all_paths = []
+ self.abandoned = False
+ self.write_started = False
+
+ def append(self, path):
+ super().append(path)
+ self.all_paths.append(path)
+
+
@contextmanager
-def capture_lease():
+def capture_lease(authority=None):
"""Hold every path captured inside this block (capture through admission commit or refusal)
against a concurrent release: an older turn's settlement or cancellation that names the same
content-addressed bytes would otherwise unlink them before the new admission holds them."""
- token = _lease_scope.set([])
+ scope = _CaptureScope(authority)
+ token = _lease_scope.set(scope)
try:
- yield
+ yield scope
finally:
drop_capture_lease()
_lease_scope.reset(token)
-def drop_capture_lease(paths=None):
+def drop_capture_lease(paths=None, *, scope=None):
"""End this scope's lease on *paths* (default: all) early, e.g. before a refused admission
releases its own fresh capture, which its lease would otherwise keep from collection."""
- held = _lease_scope.get()
+ held = scope if scope is not None else _lease_scope.get()
if not held:
return
with _lease_lock:
@@ -64,6 +78,17 @@ def drop_capture_lease(paths=None):
del _leases[path]
+async def release_abandoned_capture(scope):
+ """Collect every path published by an abandoned worker, including after its caller left."""
+ if scope.authority is None:
+ return
+ drop_capture_lease(scope=scope)
+ references = [{'path': path, 'sha256': Path(path).parent.name} for path in set(scope.all_paths)]
+ if references:
+ import asyncio
+ await asyncio.to_thread(release_unheld_media, scope.authority.db, references)
+
+
def native_inputs_store():
"""The store's path as the profile names it, unresolved. The one value ``make_user_dir`` gets:
``resolve()`` erases a link planted below the Hermes home (``cache/documents`` -> elsewhere),
diff --git a/tests/gateway/test_native_media_capture_off_loop.py b/tests/gateway/test_native_media_capture_off_loop.py
index 29bc420204e..1bbde98c411 100644
--- a/tests/gateway/test_native_media_capture_off_loop.py
+++ b/tests/gateway/test_native_media_capture_off_loop.py
@@ -1,4 +1,6 @@
"""Native media capture (copy + SHA-256 + fsyncs) runs off the owner loop, under the admission lease."""
+import asyncio
+import hashlib
import threading
import pytest
@@ -9,10 +11,92 @@ from gateway.run import GatewayRunner
from gateway.session_authority import SessionAuthority, initialize_session_authority
from gateway.session_envelope import restore_native
from hermes_state_runtime import get_session_admission
+from tests.waiting import async_wait_until
[email protected]('pause_after_publish', [False, True])
+async def test_cancelled_native_capture_releases_unadmitted_bytes(tmp_path, monkeypatch, pause_after_publish):
+ import gateway.session_envelope as envelope
+ import gateway.session_ingress_media as media
+ from plugins.platforms.whatsapp.adapter import WhatsAppAdapter
+ monkeypatch.setenv('WHATSAPP_ALLOWED_USERS', '15551234567')
+ runner = GatewayRunner(GatewayConfig(sessions_dir=tmp_path / 'sessions'))
+ adapter = WhatsAppAdapter(PlatformConfig(enabled=True, extra={'session_path': str(tmp_path / 'wa')}))
+ runner.adapters = {Platform.WHATSAPP: adapter}
+ authority = await initialize_session_authority(runner, profile_id='default', instance_id='cancel-media')
+ source_file = tmp_path / 'clip.mp4'
+ source_file.write_bytes(b'video-bytes' * 1000)
+ event = MessageEvent(text='look', message_type=MessageType.VIDEO, message_id='cancel-v1',
+ source=adapter.build_source(chat_id=CHAT, user_id=CHAT, chat_type='dm'),
+ media_urls=[str(source_file)], media_types=['video/mp4'])
+ entered, release, finished = threading.Event(), threading.Event(), threading.Event()
+ original = media.capture_native_media
+
+ def paused(paths):
+ try:
+ if not pause_after_publish:
+ entered.set()
+ assert release.wait(10)
+ references = original(paths)
+ if pause_after_publish:
+ entered.set()
+ assert release.wait(10)
+ return references
+ finally:
+ finished.set()
+
+ monkeypatch.setattr(envelope, 'capture_native_media', paused)
+ try:
+ admission = asyncio.create_task(authority.admit_native(event))
+ assert await asyncio.to_thread(entered.wait, 10)
+ admission.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(admission, 3)
+ release.set()
+ assert await asyncio.to_thread(finished.wait, 10)
+ root = media._media_root()
+ captured = root / hashlib.sha256(source_file.read_bytes()).hexdigest() / 'clip.mp4'
+ await async_wait_until(lambda: str(captured) not in media._leases and not captured.exists(),
+ timeout=3, what='abandoned capture to be collected')
+ assert str(captured) not in media._leases
+ assert not captured.exists()
+ finally:
+ release.set()
+ authority.db.close()
+
+
+async def test_native_capture_error_releases_published_bytes(tmp_path, monkeypatch):
+ import gateway.session_ingress_media as media
+ from plugins.platforms.whatsapp.adapter import WhatsAppAdapter
+ monkeypatch.setenv('WHATSAPP_ALLOWED_USERS', '15551234567')
+ runner = GatewayRunner(GatewayConfig(sessions_dir=tmp_path / 'sessions'))
+ adapter = WhatsAppAdapter(PlatformConfig(enabled=True, extra={'session_path': str(tmp_path / 'wa')}))
+ runner.adapters = {Platform.WHATSAPP: adapter}
+ authority = await initialize_session_authority(runner, profile_id='default', instance_id='error-media')
+ source_file = tmp_path / 'clip.mp4'
+ source_file.write_bytes(b'video-bytes' * 1000)
+ target = media._media_root() / hashlib.sha256(source_file.read_bytes()).hexdigest() / source_file.name
+ event = MessageEvent(text='look', message_type=MessageType.VIDEO, message_id='error-v1',
+ source=adapter.build_source(chat_id=CHAT, user_id=CHAT, chat_type='dm'),
+ media_urls=[str(source_file)], media_types=['video/mp4'])
+
+ def failed_fsync(_directory):
+ raise OSError('disk failed after publish')
+
+ monkeypatch.setattr(media, 'fsync_directory', failed_fsync)
+ try:
+ with pytest.raises(OSError, match='disk failed after publish'):
+ await authority.admit_native(event)
+ assert not target.exists()
+ assert str(target) not in media._leases
+ finally:
+ authority.db.close()
+
+
@pytest.mark.asyncio
async def test_native_media_capture_runs_off_loop_and_commits_owned_bytes(tmp_path, monkeypatch):
import gateway.session_envelope as envelope