Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tasks): emit sandbox exit status, network and cpu pressure metrics (#120942)

Alessandro Pogliaghi64a414199036
diff --git a/posthog/temporal/common/worker.py b/posthog/temporal/common/worker.py
index 74a22a0b51fc..8d854bebba2e 100644
--- a/posthog/temporal/common/worker.py
+++ b/posthog/temporal/common/worker.py
@@ -91,12 +91,16 @@ from products.logs.backend.facade.temporal import (
LogsAlertingMetricsInterceptor,
)
from products.tasks.backend.facade.temporal import (
+ TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_BUCKETS,
+ TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_METRICS,
TASKS_LATENCY_HISTOGRAM_BUCKETS,
TASKS_LATENCY_HISTOGRAM_METRICS,
TASKS_LAUNCH_PREPARATION_HISTOGRAM_BUCKETS,
TASKS_LAUNCH_PREPARATION_HISTOGRAM_METRICS,
TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS,
TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_METRICS,
+ TASKS_NETWORK_BYTES_HISTOGRAM_BUCKETS,
+ TASKS_NETWORK_BYTES_HISTOGRAM_METRICS,
TASKS_RUN_TOKENS_HISTOGRAM_BUCKETS,
TASKS_RUN_TOKENS_HISTOGRAM_METRICS,
TASKS_RUN_TURNS_HISTOGRAM_BUCKETS,
@@ -373,6 +377,12 @@ async def create_worker(
| dict(
zip(TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_METRICS, itertools.repeat(TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS))
)
+ | dict(
+ zip(
+ TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_METRICS, itertools.repeat(TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_BUCKETS)
+ )
+ )
+ | dict(zip(TASKS_NETWORK_BYTES_HISTOGRAM_METRICS, itertools.repeat(TASKS_NETWORK_BYTES_HISTOGRAM_BUCKETS)))
| dict(
zip(
EVAL_REPORTS_LATENCY_HISTOGRAM_METRICS,
diff --git a/products/tasks/backend/facade/temporal.py b/products/tasks/backend/facade/temporal.py
index 2248c51967dc..2fcb05122421 100644
--- a/products/tasks/backend/facade/temporal.py
+++ b/products/tasks/backend/facade/temporal.py
@@ -17,12 +17,16 @@ from products.tasks.backend.temporal.client import (
signal_task_followup_message,
)
from products.tasks.backend.temporal.metrics import (
+ TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_BUCKETS,
+ TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_METRICS,
TASKS_LATENCY_HISTOGRAM_BUCKETS,
TASKS_LATENCY_HISTOGRAM_METRICS,
TASKS_LAUNCH_PREPARATION_HISTOGRAM_BUCKETS,
TASKS_LAUNCH_PREPARATION_HISTOGRAM_METRICS,
TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS,
TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_METRICS,
+ TASKS_NETWORK_BYTES_HISTOGRAM_BUCKETS,
+ TASKS_NETWORK_BYTES_HISTOGRAM_METRICS,
TASKS_RUN_TOKENS_HISTOGRAM_BUCKETS,
TASKS_RUN_TOKENS_HISTOGRAM_METRICS,
TASKS_RUN_TURNS_HISTOGRAM_BUCKETS,
@@ -38,12 +42,16 @@ from products.tasks.backend.temporal.process_task.workflow import ProcessTaskWor
__all__ = [
"ACTIVITIES",
+ "TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_BUCKETS",
+ "TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_METRICS",
"TASKS_LATENCY_HISTOGRAM_BUCKETS",
"TASKS_LATENCY_HISTOGRAM_METRICS",
"TASKS_LAUNCH_PREPARATION_HISTOGRAM_BUCKETS",
"TASKS_LAUNCH_PREPARATION_HISTOGRAM_METRICS",
"TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS",
"TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_METRICS",
+ "TASKS_NETWORK_BYTES_HISTOGRAM_BUCKETS",
+ "TASKS_NETWORK_BYTES_HISTOGRAM_METRICS",
"TASKS_RUN_TOKENS_HISTOGRAM_BUCKETS",
"TASKS_RUN_TOKENS_HISTOGRAM_METRICS",
"TASKS_RUN_TURNS_HISTOGRAM_BUCKETS",
diff --git a/products/tasks/backend/logic/services/modal_sandbox.py b/products/tasks/backend/logic/services/modal_sandbox.py
index f8a619baaf61..c51e961d47e0 100644
--- a/products/tasks/backend/logic/services/modal_sandbox.py
+++ b/products/tasks/backend/logic/services/modal_sandbox.py
@@ -25,6 +25,7 @@ from django.conf import settings
import modal
import requests
from cachetools import TTLCache, cached
+from modal._utils.async_utils import synchronizer as modal_synchronizer
from modal.exception import (
ConnectionError as ModalConnectionError,
InvalidError as ModalInvalidError,
@@ -32,6 +33,7 @@ from modal.exception import (
ServiceError as ModalServiceError,
TimeoutError as ModalTimeoutError,
)
+from modal_proto import api_pb2
from python_socks import ProxyError as SocksProxyError
from semantic_version import NpmSpec
@@ -1998,6 +2000,26 @@ class ModalSandbox(AgentServerLaunchMixin):
return "timed out"
return f"exited with code {returncode}"
+ def exit_status(self) -> str | None:
+ try:
+ returncode = self._sandbox.poll()
+ except Exception as e:
+ logger.warning(f"Failed to poll sandbox {self.id} for its exit status: {e}")
+ return "unknown"
+ if returncode is None:
+ return "running"
+ try:
+ result = getattr(modal_synchronizer._translate_in(self._sandbox), "_result", None)
+ if result is not None:
+ return api_pb2.GenericResult.GenericStatus.Name(result.status).lower().removeprefix("generic_status_")
+ except Exception as e:
+ logger.warning(f"Failed to read exit status for sandbox {self.id}: {e}")
+ if returncode == 124:
+ return "timeout"
+ if returncode == 137:
+ return "terminated"
+ return "success" if returncode == 0 else "failure"
+
@property
def name(self) -> str:
return self.config.name
diff --git a/products/tasks/backend/logic/services/sandbox.py b/products/tasks/backend/logic/services/sandbox.py
index 7633bdc93853..388ad19f7e82 100644
--- a/products/tasks/backend/logic/services/sandbox.py
+++ b/products/tasks/backend/logic/services/sandbox.py
@@ -733,6 +733,9 @@ class SandboxBase(ABC):
def exit_reason(self) -> str | None:
return None
+ def exit_status(self) -> str | None:
+ return None
+
def read_agent_server_session_init_ms(self) -> int | None:
return None
diff --git a/products/tasks/backend/logic/services/sandbox_teardown_stats.py b/products/tasks/backend/logic/services/sandbox_teardown_stats.py
new file mode 100644
index 000000000000..cfbf1f86adcc
--- /dev/null
+++ b/products/tasks/backend/logic/services/sandbox_teardown_stats.py
@@ -0,0 +1,105 @@
+from posthog.dataclasses import frozen
+
+from products.tasks.backend.logic.services.memory_watchdog import build_memory_watchdog_read_command
+
+SANDBOX_TEARDOWN_STATS_SEPARATOR = "__posthog_sandbox_teardown_stats__"
+
+
+@frozen
+class SandboxTeardownStats:
+ rx_bytes: int | None
+ tx_bytes: int | None
+ cpu_pressure_ratio: float | None
+
+
+def build_sandbox_teardown_stats_command() -> str:
+ return (
+ f"cat /proc/net/dev 2>/dev/null || true; echo {SANDBOX_TEARDOWN_STATS_SEPARATOR}; "
+ f"cat /proc/pressure/cpu 2>/dev/null || true; echo {SANDBOX_TEARDOWN_STATS_SEPARATOR}; "
+ "cat /proc/uptime 2>/dev/null || true"
+ )
+
+
+def build_sandbox_teardown_read_command() -> str:
+ return (
+ f"{build_memory_watchdog_read_command()}; echo {SANDBOX_TEARDOWN_STATS_SEPARATOR}; "
+ f"{build_sandbox_teardown_stats_command()}"
+ )
+
+
+@frozen
+class SandboxTeardownReadOutput:
+ watchdog_output: str
+ stats_output: str
+
+
+@frozen
+class SandboxNetworkBytes:
+ rx_bytes: int | None
+ tx_bytes: int | None
+
+
+def split_sandbox_teardown_read_output(stdout: str) -> SandboxTeardownReadOutput:
+ watchdog_output, _, stats_output = stdout.partition(SANDBOX_TEARDOWN_STATS_SEPARATOR)
+ return SandboxTeardownReadOutput(watchdog_output=watchdog_output, stats_output=stats_output)
+
+
+def _parse_network_bytes(net_dev: str) -> SandboxNetworkBytes:
+ rx_total = 0
+ tx_total = 0
+ found = False
+ for line in net_dev.splitlines():
+ if ":" not in line:
+ continue
+ interface, _, counters = line.partition(":")
+ if interface.strip() == "lo":
+ continue
+ fields = counters.split()
+ if len(fields) < 9:
+ continue
+ try:
+ rx_bytes = int(fields[0])
+ tx_bytes = int(fields[8])
+ except ValueError:
+ continue
+ rx_total += rx_bytes
+ tx_total += tx_bytes
+ found = True
+ if not found:
+ return SandboxNetworkBytes(rx_bytes=None, tx_bytes=None)
+ return SandboxNetworkBytes(rx_bytes=rx_total, tx_bytes=tx_total)
+
+
+def _parse_cpu_pressure_ratio(pressure: str, uptime: str) -> float | None:
+ stall_usec: int | None = None
+ for line in pressure.splitlines():
+ parts = line.split()
+ if not parts or parts[0] != "some":
+ continue
+ for part in parts[1:]:
+ key, _, value = part.partition("=")
+ if key == "total":
+ try:
+ stall_usec = int(value)
+ except ValueError:
+ return None
+ if stall_usec is None:
+ return None
+ try:
+ uptime_seconds = float(uptime.split()[0])
+ except (IndexError, ValueError):
+ return None
+ if uptime_seconds <= 0:
+ return None
+ return min(max(stall_usec / (uptime_seconds * 1e6), 0.0), 1.0)
+
+
+def parse_sandbox_teardown_stats(output: str) -> SandboxTeardownStats:
+ sections = output.split(SANDBOX_TEARDOWN_STATS_SEPARATOR)
+ net_dev, pressure, uptime = [*sections, "", "", ""][:3]
+ network = _parse_network_bytes(net_dev)
+ return SandboxTeardownStats(
+ rx_bytes=network.rx_bytes,
+ tx_bytes=network.tx_bytes,
+ cpu_pressure_ratio=_parse_cpu_pressure_ratio(pressure, uptime),
+ )
diff --git a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
index af4fec736ad9..ca3dc98c830b 100644
--- a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
+++ b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
@@ -22,6 +22,7 @@ from modal.exception import (
ServiceError as ModalServiceError,
TimeoutError as ModalTimeoutError,
)
+from modal_proto import api_pb2
from parameterized import parameterized
from python_socks import ProxyError as SocksProxyError
from requests.exceptions import (
@@ -1607,6 +1608,25 @@ class TestModalSandboxAgentServerStartupHelpers:
sandbox._sandbox = MagicMock()
return sandbox
+ @pytest.mark.parametrize(
+ "poll_result,status,expected",
+ [
+ (None, api_pb2.GenericResult.GENERIC_STATUS_UNSPECIFIED, "running"),
+ (RuntimeError("provider unavailable"), api_pb2.GenericResult.GENERIC_STATUS_UNSPECIFIED, "unknown"),
+ (137, api_pb2.GenericResult.GENERIC_STATUS_MEMORY_MANAGER_EVICTION, "memory_manager_eviction"),
+ (0, api_pb2.GenericResult.GENERIC_STATUS_IDLE_TIMEOUT, "idle_timeout"),
+ ],
+ )
+ def test_exit_status_reads_modal_generic_status(self, poll_result, status, expected):
+ sandbox = self._make_sandbox()
+ if isinstance(poll_result, Exception):
+ sandbox._sandbox.poll.side_effect = poll_result
+ else:
+ sandbox._sandbox.poll.return_value = poll_result
+ sandbox._sandbox._result.status = status
+
+ assert sandbox.exit_status() == expected
+
def test_free_agent_server_port_terminates_existing_process(self):
sandbox = self._make_sandbox()
sandbox.execute = MagicMock(return_value=ExecutionResult(stdout="", stderr="", exit_code=0, error=None))
diff --git a/products/tasks/backend/logic/services/tests/test_sandbox_teardown_stats.py b/products/tasks/backend/logic/services/tests/test_sandbox_teardown_stats.py
new file mode 100644
index 000000000000..a2b785d67b4f
--- /dev/null
+++ b/products/tasks/backend/logic/services/tests/test_sandbox_teardown_stats.py
@@ -0,0 +1,46 @@
+import pytest
+
+from products.tasks.backend.logic.services.sandbox_teardown_stats import (
+ SANDBOX_TEARDOWN_STATS_SEPARATOR,
+ SandboxTeardownStats,
+ parse_sandbox_teardown_stats,
+)
+
+NET_DEV = """Inter-| Receive | Transmit
+ face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed
+ lo: 9999999 100 0 0 0 0 0 0 9999999 100 0 0 0 0 0 0
+ eth0: 1000000 2000 0 0 0 0 0 0 300000 1000 0 0 0 0 0 0
+ eth1: 500000 100 0 0 0 0 0 0 20000 50 0 0 0 0 0 0
+"""
+PRESSURE = "some avg10=0.00 avg60=0.00 avg300=0.00 total=25000000\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n"
+UPTIME = "100.00 350.00\n"
+
+
+def _output(net_dev: str, pressure: str, uptime: str) -> str:
+ return f"{net_dev}\n{SANDBOX_TEARDOWN_STATS_SEPARATOR}\n{pressure}\n{SANDBOX_TEARDOWN_STATS_SEPARATOR}\n{uptime}"
+
+
+ "output,expected",
+ [
+ (
+ _output(NET_DEV, PRESSURE, UPTIME),
+ SandboxTeardownStats(rx_bytes=1_500_000, tx_bytes=320_000, cpu_pressure_ratio=0.25),
+ ),
+ (
+ _output(NET_DEV, "", UPTIME),
+ SandboxTeardownStats(rx_bytes=1_500_000, tx_bytes=320_000, cpu_pressure_ratio=None),
+ ),
+ (
+ _output(NET_DEV, "some avg10=0.00 total=500000000\n", UPTIME),
+ SandboxTeardownStats(rx_bytes=1_500_000, tx_bytes=320_000, cpu_pressure_ratio=1.0),
+ ),
+ (
+ "garbage: not numbers\nmore garbage",
+ SandboxTeardownStats(rx_bytes=None, tx_bytes=None, cpu_pressure_ratio=None),
+ ),
+ ],
+ ids=["full", "missing_psi", "clamped", "garbage"],
+)
+def test_parse_sandbox_teardown_stats(output: str, expected: SandboxTeardownStats) -> None:
+ assert parse_sandbox_teardown_stats(output) == expected
diff --git a/products/tasks/backend/temporal/metrics.py b/products/tasks/backend/temporal/metrics.py
index d04dfd69282f..56d25d8cd822 100644
--- a/products/tasks/backend/temporal/metrics.py
+++ b/products/tasks/backend/temporal/metrics.py
@@ -132,6 +132,21 @@ TASKS_RUN_TURNS_HISTOGRAM_BUCKETS = [
TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_METRICS = ("tasks_sandbox_memory_peak_ratio",)
TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1.0]
+TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_METRICS = ("tasks_sandbox_cpu_pressure_ratio",)
+TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_BUCKETS = TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS
+
+TASKS_NETWORK_BYTES_HISTOGRAM_METRICS = ("tasks_sandbox_network_bytes",)
+TASKS_NETWORK_BYTES_HISTOGRAM_BUCKETS = [
+ 1_048_576.0,
+ 4_194_304.0,
+ 16_777_216.0,
+ 67_108_864.0,
+ 268_435_456.0,
+ 1_073_741_824.0,
+ 4_294_967_296.0,
+ 17_179_869_184.0,
+]
+
_RUN_TOKEN_KINDS = {
"input": "input_tokens",
"output": "output_tokens",
@@ -388,6 +403,36 @@ def record_memory_peak_ratio(ratio: float) -> None:
pass
+def increment_sandbox_exit_status(status: str, runtime: str) -> None:
+ try:
+ _metric_meter({"status": status, "runtime": runtime}).create_counter(
+ "tasks_sandbox_exit_status",
+ "Sandbox exit status read at cleanup, running when the sandbox was still alive, unknown when the poll failed",
+ ).add(1)
+ except Exception:
+ pass
+
+
+def record_sandbox_network_bytes(direction: str, value: int) -> None:
+ try:
+ _metric_meter({"direction": direction}).create_histogram(
+ "tasks_sandbox_network_bytes",
+ "Sandbox network bytes over non-loopback interfaces, read at teardown",
+ ).record(value)
+ except Exception:
+ pass
+
+
+def record_sandbox_cpu_pressure_ratio(ratio: float) -> None:
+ try:
+ _metric_meter().create_histogram_float(
+ "tasks_sandbox_cpu_pressure_ratio",
+ "Fraction of sandbox uptime with some task stalled on CPU, read at teardown",
+ ).record(ratio)
+ except Exception:
+ pass
+
+
def increment_tool_call_only_heartbeat() -> None:
try:
_metric_meter().create_counter(
diff --git a/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py b/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py
index 24c6f0ba422d..ed822d2d37bd 100644
--- a/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py
+++ b/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py
@@ -20,6 +20,7 @@ from products.tasks.backend.logic.stream.redis_stream import (
)
from products.tasks.backend.models import SandboxSession, TaskRun
from products.tasks.backend.redis import run_uses_dedicated_stream
+from products.tasks.backend.temporal.metrics import increment_sandbox_exit_status, sandbox_runtime_label
from products.tasks.backend.temporal.observability import log_activity_execution
logger = logging.getLogger(__name__)
@@ -93,6 +94,17 @@ def cleanup_sandbox_now(input: CleanupSandboxInput) -> None:
exc_info=True,
)
+ try:
+ exit_status = sandbox.exit_status()
+ if exit_status is not None:
+ session = (
+ SandboxSession.objects.unscoped().filter(sandbox_id=input.sandbox_id).only("vm_runtime").first()
+ )
+ runtime = sandbox_runtime_label(session.vm_runtime) if session is not None else "unknown"
+ increment_sandbox_exit_status(exit_status, runtime)
+ except Exception:
+ logger.warning("cleanup_sandbox_exit_status_failed", extra={"sandbox_id": input.sandbox_id}, exc_info=True)
+
cpu_usage_usec, cpu_usage_measured_at = measure_sandbox_cpu_usage(sandbox)
billed_cpu_usage_usec = measure_sandbox_billed_cpu_usage(sandbox)
diff --git a/products/tasks/backend/temporal/process_task/activities/read_sandbox_logs.py b/products/tasks/backend/temporal/process_task/activities/read_sandbox_logs.py
index 9bc88b61c5d5..3788cf1bf8c2 100644
--- a/products/tasks/backend/temporal/process_task/activities/read_sandbox_logs.py
+++ b/products/tasks/backend/temporal/process_task/activities/read_sandbox_logs.py
@@ -8,15 +8,19 @@ from temporalio import activity
from posthog.temporal.common.utils import asyncify
from products.tasks.backend.exceptions import SandboxNotRunningError
-from products.tasks.backend.logic.services.memory_watchdog import (
- build_memory_watchdog_read_command,
- parse_memory_watchdog_read_output,
-)
+from products.tasks.backend.logic.services.memory_watchdog import parse_memory_watchdog_read_output
from products.tasks.backend.logic.services.sandbox import SandboxBase, get_sandbox_class_for_sandbox_id
+from products.tasks.backend.logic.services.sandbox_teardown_stats import (
+ build_sandbox_teardown_read_command,
+ parse_sandbox_teardown_stats,
+ split_sandbox_teardown_read_output,
+)
from products.tasks.backend.temporal.metrics import (
increment_memory_watchdog_events,
increment_memory_watchdog_teardown,
record_memory_peak_ratio,
+ record_sandbox_cpu_pressure_ratio,
+ record_sandbox_network_bytes,
)
from products.tasks.backend.temporal.observability import log_activity_execution
@@ -34,9 +38,8 @@ class ReadSandboxLogsInput:
SANDBOX_TERMINATED_MESSAGE = "Sandbox terminated before logs could be captured; no agent-server logs available."
-def _report_memory_watchdog(sandbox: SandboxBase, sandbox_id: str, run_id: str | None) -> None:
- result = sandbox.execute(build_memory_watchdog_read_command(), timeout_seconds=10)
- summary = parse_memory_watchdog_read_output(result.stdout, time.time())
+def _report_memory_watchdog(stdout: str, sandbox_id: str, run_id: str | None) -> None:
+ summary = parse_memory_watchdog_read_output(stdout, time.time())
logger.info(
f"Sandbox memory watchdog summary run_id={run_id} sandbox_id={sandbox_id} kills={summary.kill_count} "
f"no_target={summary.no_target} pressure_episodes={summary.pressure_episodes} "
@@ -50,6 +53,27 @@ def _report_memory_watchdog(sandbox: SandboxBase, sandbox_id: str, run_id: str |
record_memory_peak_ratio(summary.peak_ratio)
+def _report_teardown_stats(stdout: str, sandbox_id: str, run_id: str | None) -> None:
+ stats = parse_sandbox_teardown_stats(stdout)
+ logger.info(
+ f"Sandbox teardown stats run_id={run_id} sandbox_id={sandbox_id} rx_bytes={stats.rx_bytes} "
+ f"tx_bytes={stats.tx_bytes} cpu_pressure_ratio={stats.cpu_pressure_ratio}"
+ )
+ if stats.rx_bytes is not None:
+ record_sandbox_network_bytes("rx", stats.rx_bytes)
+ if stats.tx_bytes is not None:
+ record_sandbox_network_bytes("tx", stats.tx_bytes)
+ if stats.cpu_pressure_ratio is not None:
+ record_sandbox_cpu_pressure_ratio(stats.cpu_pressure_ratio)
+
+
+def _report_teardown(sandbox: SandboxBase, sandbox_id: str, run_id: str | None) -> None:
+ result = sandbox.execute(build_sandbox_teardown_read_command(), timeout_seconds=10)
+ output = split_sandbox_teardown_read_output(result.stdout)
+ _report_memory_watchdog(output.watchdog_output, sandbox_id, run_id)
+ _report_teardown_stats(output.stats_output, sandbox_id, run_id)
+
+
@activity.defn
@asyncify
def read_sandbox_logs(input: ReadSandboxLogsInput) -> str:
@@ -108,9 +132,9 @@ def read_sandbox_logs(input: ReadSandboxLogsInput) -> str:
logger.debug("agentsh audit query failed for sandbox %s", input.sandbox_id, exc_info=True)
try:
- _report_memory_watchdog(sandbox, input.sandbox_id, input.run_id)
+ _report_teardown(sandbox, input.sandbox_id, input.run_id)
except Exception:
- logger.warning("Memory watchdog summary failed for sandbox %s", input.sandbox_id, exc_info=True)
+ logger.warning("Teardown stats failed for sandbox %s", input.sandbox_id, exc_info=True)
return logs
except SandboxNotRunningError:
diff --git a/products/tasks/backend/temporal/process_task/tests/test_read_sandbox_logs.py b/products/tasks/backend/temporal/process_task/tests/test_read_sandbox_logs.py
index bbd0133ace79..e45e145ebed8 100644
--- a/products/tasks/backend/temporal/process_task/tests/test_read_sandbox_logs.py
+++ b/products/tasks/backend/temporal/process_task/tests/test_read_sandbox_logs.py
@@ -7,8 +7,8 @@ from products.tasks.backend.logic.services.memory_watchdog import (
MEMORY_WATCHDOG_EVENTS_SEPARATOR,
MEMORY_WATCHDOG_NOW_PREFIX,
MEMORY_WATCHDOG_STATE_SEPARATOR,
- build_memory_watchdog_read_command,
)
+from products.tasks.backend.logic.services.sandbox_teardown_stats import build_sandbox_teardown_read_command
from products.tasks.backend.temporal.process_task.activities.read_sandbox_logs import (
SANDBOX_TERMINATED_MESSAGE,
ReadSandboxLogsInput,
@@ -71,7 +71,7 @@ def test_reports_memory_watchdog_kills_to_metrics():
sandbox = MagicMock()
sandbox.is_running.return_value = True
sandbox.execute.side_effect = lambda command, **_: MagicMock(
- stdout=watchdog_output if command == build_memory_watchdog_read_command() else ""
+ stdout=watchdog_output if command == build_sandbox_teardown_read_command() else ""
)
module = "products.tasks.backend.temporal.process_task.activities.read_sandbox_logs"
diff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.py
index 1b2a81b7a548..93a128581d8e 100644
--- a/products/tasks/backend/tests/test_api.py
+++ b/products/tasks/backend/tests/test_api.py
@@ -11624,6 +11624,7 @@ class TestTaskRunCancelAPI(BaseTaskAPITest):
sandbox.method_calls,
[
call.stop_agent_server(),
+ call.exit_status(),
call.read_cpu_usage_usec(),
call.read_billed_cpu_usage_usec(),
call.destroy(),
@@ -11668,6 +11669,7 @@ class TestTaskRunCancelAPI(BaseTaskAPITest):
sandbox.method_calls,
[
call.stop_agent_server(),
+ call.exit_status(),
call.read_cpu_usage_usec(),
call.read_billed_cpu_usage_usec(),
call.destroy(),
@@ -11721,6 +11723,7 @@ class TestTaskRunCancelAPI(BaseTaskAPITest):
sandbox.method_calls,
[
call.stop_agent_server(),
+ call.exit_status(),
call.read_cpu_usage_usec(),
call.read_billed_cpu_usage_usec(),
call.destroy(),