Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(tasks): stop watchdog survivors after memory recovers (#119381)

Alessandro Pogliaghid31e132b297b
diff --git a/products/tasks/backend/logic/services/tests/test_memory_watchdog.py b/products/tasks/backend/logic/services/tests/test_memory_watchdog.py
index 44a6a996e874..3521f2f44dcc 100644
--- a/products/tasks/backend/logic/services/tests/test_memory_watchdog.py
+++ b/products/tasks/backend/logic/services/tests/test_memory_watchdog.py
@@ -406,3 +406,33 @@ def test_read_command_caps_what_it_pulls_out_of_the_sandbox(tmp_path: Path) -> N
assert result.returncode == 0
assert len(result.stdout) < 300 * 1024
assert result.stdout.rstrip().endswith(b'{"event": "kill", "pid": 19999}')
+
+
+def test_enforce_kills_survivors_after_memory_drops_below_trigger(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ signals: list[tuple[tuple[int, ...], str]] = []
+ alive = {40, 41, 42}
+
+ def record_signal(pids: dict[int, int] | tuple[int, ...], sig: sidecar.signal.Signals) -> None:
+ signals.append((tuple(pids), sig.name))
+ if sig == sidecar.signal.SIGTERM:
+ alive.difference_update({40, 42})
+ else:
+ alive.difference_update(pids)
+
+ monkeypatch.setattr(sidecar, "EVENTS_PATH", tmp_path / "events.jsonl")
+ monkeypatch.setattr(sidecar, "TERM_GRACE_SECONDS", 0.0)
+ monkeypatch.setattr(sidecar, "list_processes", lambda: SANDBOX_PROCESSES)
+ monkeypatch.setattr(sidecar, "read_pids", lambda path: (20,))
+ monkeypatch.setattr(sidecar, "read_pid", lambda path: 10)
+ monkeypatch.setattr(sidecar, "_start_time", lambda pid: pid)
+ monkeypatch.setattr(sidecar, "_alive_matching", lambda identities: tuple(pid for pid in identities if pid in alive))
+ monkeypatch.setattr(sidecar, "signal_pids", record_signal)
+ monkeypatch.setattr(sidecar, "read_memory", lambda: sidecar.MemoryReading(10 * GIB, 16 * GIB, "cgroup_v2"))
+ watchdog = sidecar.Watchdog(sidecar.MemoryReading(14 * GIB, 16 * GIB, "cgroup_v2"))
+
+ watchdog.enforce(0.0)
+
+ assert signals == [((42, 41, 40), "SIGTERM"), ((41,), "SIGKILL")]
+ assert json.loads((tmp_path / "events.jsonl").read_text())["signal"] == "SIGKILL"
diff --git a/products/tasks/backend/sandbox/images/memory_watchdog.py b/products/tasks/backend/sandbox/images/memory_watchdog.py
index 32fd264af64b..b30e703ce0f8 100644
--- a/products/tasks/backend/sandbox/images/memory_watchdog.py
+++ b/products/tasks/backend/sandbox/images/memory_watchdog.py
@@ -397,13 +397,11 @@ class Watchdog:
self.peak_current = max(self.peak_current, reading.current)
return reading
- def wait_for_release(self, identities: dict[int, int]) -> None:
+ def wait_for_exit(self, identities: dict[int, int]) -> None:
deadline = time.monotonic() + TERM_GRACE_SECONDS
while time.monotonic() < deadline and _alive_matching(identities):
time.sleep(POLL_SECONDS)
- reading = self.observe()
- if reading is not None and not should_trigger(reading.current, reading.limit, TRIGGER_RATIO):
- return
+ self.observe()
def wait_below_trigger(self) -> bool:
deadline = time.monotonic() + TERM_GRACE_SECONDS
@@ -437,10 +435,9 @@ class Watchdog:
identities = {pid: start for pid in tree.pids if (start := _start_time(pid)) is not None}
signal_pids(identities, signal.SIGTERM)
final_signal = signal.SIGTERM
- self.wait_for_release(identities)
- reading = self.observe()
+ self.wait_for_exit(identities)
remaining = _alive_matching(identities)
- if reading is not None and should_trigger(reading.current, reading.limit, TRIGGER_RATIO) and remaining:
+ if remaining:
signal_pids(remaining, signal.SIGKILL)
final_signal = signal.SIGKILL
released = self.wait_below_trigger()