Python
fix(observability): stop the Phase 3 background work before the stores close (T3-3)
Devika Verma2a903b725859
diff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex e999b35d..7b7a034e 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -453,6 +453,8 @@ class SessionManager:import concurrent.futuresself._scope_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="obs-scopes")+ # T3-3: set when the app stops, so the start-up checks stop between sessions+ self._obs_closing = threading.Event()self.recorder.add_listener(self._scopes_on_turn_end)# P3B-5: a team's Phase 3 numbers and task checks, kept until a member's recording,# the board or the journal changes (team_id -> what they were worked out from, result).@@ -7294,6 +7296,11 @@ class SessionManager:logger.debug("sandbox reap at start failed", exc_info=True)async def aclose(self) -> None:+ # T3-3: the Phase 3 background work stops before the stores close. The scope pool's+ # thread is not a daemon, so Python waits for it at exit, and it would run every+ # queued check on a closed store first; the start-up checks stop between sessions.+ self._obs_closing.set()+ self._scope_pool.shutdown(wait=False, cancel_futures=True)# Sandboxes first: each holds a container or a sandboxed process that nobody else# would remove once this server is gone.await asyncio.to_thread(self._close_all_sandboxes)@@ -9054,6 +9061,8 @@ class SessionManager:never lights the dot. Returns how many sessions were checked."""sids = self.recordings_store.sessions_needing_check(recording_signals.version())for sid in sids:+ if self._obs_closing.is_set():+ break # the app is stopping; the rest are checked at the next starttry:self._recording_issues(sid, self.recordings_store.load_recording(sid))except Exception: # noqa: BLE001 - one bad recording must not stop the rest@@ -9070,6 +9079,8 @@ class SessionManager:if not ids or ids == last:return done # nothing left, or a batch that cannot be written (logged)last = ids+ if self._obs_closing.is_set():+ return done # the app is stopping; the rest are filled at the next startfor tid in ids:self.recordings_store.refresh_usage(tid)self.recordings_store.flush()diff --git a/tests/test_observability_phase3_manual_fixes.py b/tests/test_observability_phase3_manual_fixes.pyindex 7d81507b..c0f0b89a 100644--- a/tests/test_observability_phase3_manual_fixes.py+++ b/tests/test_observability_phase3_manual_fixes.py@@ -149,3 +149,35 @@ def test_the_ratings_card_counts_reasons_and_lists_the_down_turns(tmp_path):one = usage_api.ratings(store, start=RECENT - 1, end=RECENT + 60_000, persona_id="sec")assert (one["up"], one["down"]) == (1, 1) and one["down_turns"][0]["turn_id"] == "T2"store.close()+++# -- T3-3 -----------------------------------------------------------------------------------------+def test_stopping_the_app_drops_queued_scope_checks_and_stops_the_start_up_checks(tmp_path, monkeypatch):+ import asyncio+ import threading as _threading++ from coworker.server import SessionManager++ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))+ manager = SessionManager(workspace=tmp_path)+ gate, ran = _threading.Event(), []++ def check(sid, tid):+ ran.append(tid)+ gate.wait(5)++ monkeypatch.setattr(manager, "run_scope_checks", check)+ for i in range(20):+ manager._scopes_on_turn_end({"type": "turn_ended", "turn_id": f"T{i}", "session_id": "s"})+ asyncio.run(manager.aclose())+ gate.set()+ manager._scope_pool.shutdown(wait=True)+ assert len(ran) <= 1 # the one running finishes; the 19 queued are dropped+ # a turn that ends after the stop is not queued+ manager._scopes_on_turn_end({"type": "turn_ended", "turn_id": "late", "session_id": "s"})+ assert "late" not in ran+ # the start-up checks stop between sessions+ monkeypatch.setattr(manager.recordings_store, "sessions_needing_check", lambda v: ["a", "b"])+ loaded = []+ monkeypatch.setattr(manager.recordings_store, "load_recording", lambda sid: loaded.append(sid))+ assert manager.recheck_issues() == 2 and loaded == []