:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(tasks): stop watchdog survivors after memory recovers (#119381)
Alessandro Pogliaghid31e132b297b
diff --git a/products/tasks/backend/logic/services/tests/test_memory_watchdog.py b/products/tasks/backend/logic/services/tests/test_memory_watchdog.pyindex 44a6a996e874..3521f2f44dcc 100644--- a/products/tasks/backend/logic/services/tests/test_memory_watchdog.py+++ b/products/tasks/backend/logic/services/tests/test_memory_watchdog.py@@ -406,3 +406,33 @@ def test_read_command_caps_what_it_pulls_out_of_the_sandbox(tmp_path: Path) -> Nassert result.returncode == 0assert len(result.stdout) < 300 * 1024assert result.stdout.rstrip().endswith(b'{"event": "kill", "pid": 19999}')+++def test_enforce_kills_survivors_after_memory_drops_below_trigger(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ signals: list[tuple[tuple[int, ...], str]] = []+ alive = {40, 41, 42}++ def record_signal(pids: dict[int, int] | tuple[int, ...], sig: sidecar.signal.Signals) -> None:+ signals.append((tuple(pids), sig.name))+ if sig == sidecar.signal.SIGTERM:+ alive.difference_update({40, 42})+ else:+ alive.difference_update(pids)++ monkeypatch.setattr(sidecar, "EVENTS_PATH", tmp_path / "events.jsonl")+ monkeypatch.setattr(sidecar, "TERM_GRACE_SECONDS", 0.0)+ monkeypatch.setattr(sidecar, "list_processes", lambda: SANDBOX_PROCESSES)+ monkeypatch.setattr(sidecar, "read_pids", lambda path: (20,))+ monkeypatch.setattr(sidecar, "read_pid", lambda path: 10)+ monkeypatch.setattr(sidecar, "_start_time", lambda pid: pid)+ monkeypatch.setattr(sidecar, "_alive_matching", lambda identities: tuple(pid for pid in identities if pid in alive))+ monkeypatch.setattr(sidecar, "signal_pids", record_signal)+ monkeypatch.setattr(sidecar, "read_memory", lambda: sidecar.MemoryReading(10 * GIB, 16 * GIB, "cgroup_v2"))+ watchdog = sidecar.Watchdog(sidecar.MemoryReading(14 * GIB, 16 * GIB, "cgroup_v2"))++ watchdog.enforce(0.0)++ assert signals == [((42, 41, 40), "SIGTERM"), ((41,), "SIGKILL")]+ assert json.loads((tmp_path / "events.jsonl").read_text())["signal"] == "SIGKILL"diff --git a/products/tasks/backend/sandbox/images/memory_watchdog.py b/products/tasks/backend/sandbox/images/memory_watchdog.pyindex 32fd264af64b..b30e703ce0f8 100644--- a/products/tasks/backend/sandbox/images/memory_watchdog.py+++ b/products/tasks/backend/sandbox/images/memory_watchdog.py@@ -397,13 +397,11 @@ class Watchdog:self.peak_current = max(self.peak_current, reading.current)return reading- def wait_for_release(self, identities: dict[int, int]) -> None:+ def wait_for_exit(self, identities: dict[int, int]) -> None:deadline = time.monotonic() + TERM_GRACE_SECONDSwhile time.monotonic() < deadline and _alive_matching(identities):time.sleep(POLL_SECONDS)- reading = self.observe()- if reading is not None and not should_trigger(reading.current, reading.limit, TRIGGER_RATIO):- return+ self.observe()def wait_below_trigger(self) -> bool:deadline = time.monotonic() + TERM_GRACE_SECONDS@@ -437,10 +435,9 @@ class Watchdog:identities = {pid: start for pid in tree.pids if (start := _start_time(pid)) is not None}signal_pids(identities, signal.SIGTERM)final_signal = signal.SIGTERM- self.wait_for_release(identities)- reading = self.observe()+ self.wait_for_exit(identities)remaining = _alive_matching(identities)- if reading is not None and should_trigger(reading.current, reading.limit, TRIGGER_RATIO) and remaining:+ if remaining:signal_pids(remaining, signal.SIGKILL)final_signal = signal.SIGKILLreleased = self.wait_below_trigger()