Skip to content
Python

fix(observability): stop the Phase 3 background work before the stores close (T3-3)

Devika Verma2a903b725859
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index e999b35d..7b7a034e 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -453,6 +453,8 @@ class SessionManager:
import concurrent.futures
self._scope_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="obs-scopes")
+ # T3-3: set when the app stops, so the start-up checks stop between sessions
+ self._obs_closing = threading.Event()
self.recorder.add_listener(self._scopes_on_turn_end)
# P3B-5: a team's Phase 3 numbers and task checks, kept until a member's recording,
# the board or the journal changes (team_id -> what they were worked out from, result).
@@ -7294,6 +7296,11 @@ class SessionManager:
logger.debug("sandbox reap at start failed", exc_info=True)
async def aclose(self) -> None:
+ # T3-3: the Phase 3 background work stops before the stores close. The scope pool's
+ # thread is not a daemon, so Python waits for it at exit, and it would run every
+ # queued check on a closed store first; the start-up checks stop between sessions.
+ self._obs_closing.set()
+ self._scope_pool.shutdown(wait=False, cancel_futures=True)
# Sandboxes first: each holds a container or a sandboxed process that nobody else
# would remove once this server is gone.
await asyncio.to_thread(self._close_all_sandboxes)
@@ -9054,6 +9061,8 @@ class SessionManager:
never lights the dot. Returns how many sessions were checked."""
sids = self.recordings_store.sessions_needing_check(recording_signals.version())
for sid in sids:
+ if self._obs_closing.is_set():
+ break # the app is stopping; the rest are checked at the next start
try:
self._recording_issues(sid, self.recordings_store.load_recording(sid))
except Exception: # noqa: BLE001 - one bad recording must not stop the rest
@@ -9070,6 +9079,8 @@ class SessionManager:
if not ids or ids == last:
return done # nothing left, or a batch that cannot be written (logged)
last = ids
+ if self._obs_closing.is_set():
+ return done # the app is stopping; the rest are filled at the next start
for tid in ids:
self.recordings_store.refresh_usage(tid)
self.recordings_store.flush()
diff --git a/tests/test_observability_phase3_manual_fixes.py b/tests/test_observability_phase3_manual_fixes.py
index 7d81507b..c0f0b89a 100644
--- a/tests/test_observability_phase3_manual_fixes.py
+++ b/tests/test_observability_phase3_manual_fixes.py
@@ -149,3 +149,35 @@ def test_the_ratings_card_counts_reasons_and_lists_the_down_turns(tmp_path):
one = usage_api.ratings(store, start=RECENT - 1, end=RECENT + 60_000, persona_id="sec")
assert (one["up"], one["down"]) == (1, 1) and one["down_turns"][0]["turn_id"] == "T2"
store.close()
+
+
+# -- T3-3 -----------------------------------------------------------------------------------------
+def test_stopping_the_app_drops_queued_scope_checks_and_stops_the_start_up_checks(tmp_path, monkeypatch):
+ import asyncio
+ import threading as _threading
+
+ from coworker.server import SessionManager
+
+ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
+ manager = SessionManager(workspace=tmp_path)
+ gate, ran = _threading.Event(), []
+
+ def check(sid, tid):
+ ran.append(tid)
+ gate.wait(5)
+
+ monkeypatch.setattr(manager, "run_scope_checks", check)
+ for i in range(20):
+ manager._scopes_on_turn_end({"type": "turn_ended", "turn_id": f"T{i}", "session_id": "s"})
+ asyncio.run(manager.aclose())
+ gate.set()
+ manager._scope_pool.shutdown(wait=True)
+ assert len(ran) <= 1 # the one running finishes; the 19 queued are dropped
+ # a turn that ends after the stop is not queued
+ manager._scopes_on_turn_end({"type": "turn_ended", "turn_id": "late", "session_id": "s"})
+ assert "late" not in ran
+ # the start-up checks stop between sessions
+ monkeypatch.setattr(manager.recordings_store, "sessions_needing_check", lambda v: ["a", "b"])
+ loaded = []
+ monkeypatch.setattr(manager.recordings_store, "load_recording", lambda sid: loaded.append(sid))
+ assert manager.recheck_issues() == 2 and loaded == []