:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(tasks): emit sandbox exit status, network and cpu pressure metrics (#120942)
Alessandro Pogliaghi64a414199036
diff --git a/posthog/temporal/common/worker.py b/posthog/temporal/common/worker.pyindex 74a22a0b51fc..8d854bebba2e 100644--- a/posthog/temporal/common/worker.py+++ b/posthog/temporal/common/worker.py@@ -91,12 +91,16 @@ from products.logs.backend.facade.temporal import (LogsAlertingMetricsInterceptor,)from products.tasks.backend.facade.temporal import (+ TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_BUCKETS,+ TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_METRICS,TASKS_LATENCY_HISTOGRAM_BUCKETS,TASKS_LATENCY_HISTOGRAM_METRICS,TASKS_LAUNCH_PREPARATION_HISTOGRAM_BUCKETS,TASKS_LAUNCH_PREPARATION_HISTOGRAM_METRICS,TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS,TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_METRICS,+ TASKS_NETWORK_BYTES_HISTOGRAM_BUCKETS,+ TASKS_NETWORK_BYTES_HISTOGRAM_METRICS,TASKS_RUN_TOKENS_HISTOGRAM_BUCKETS,TASKS_RUN_TOKENS_HISTOGRAM_METRICS,TASKS_RUN_TURNS_HISTOGRAM_BUCKETS,@@ -373,6 +377,12 @@ async def create_worker(| dict(zip(TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_METRICS, itertools.repeat(TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS)))+ | dict(+ zip(+ TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_METRICS, itertools.repeat(TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_BUCKETS)+ )+ )+ | dict(zip(TASKS_NETWORK_BYTES_HISTOGRAM_METRICS, itertools.repeat(TASKS_NETWORK_BYTES_HISTOGRAM_BUCKETS)))| dict(zip(EVAL_REPORTS_LATENCY_HISTOGRAM_METRICS,diff --git a/products/tasks/backend/facade/temporal.py b/products/tasks/backend/facade/temporal.pyindex 2248c51967dc..2fcb05122421 100644--- a/products/tasks/backend/facade/temporal.py+++ b/products/tasks/backend/facade/temporal.py@@ -17,12 +17,16 @@ from products.tasks.backend.temporal.client import (signal_task_followup_message,)from products.tasks.backend.temporal.metrics import (+ TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_BUCKETS,+ TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_METRICS,TASKS_LATENCY_HISTOGRAM_BUCKETS,TASKS_LATENCY_HISTOGRAM_METRICS,TASKS_LAUNCH_PREPARATION_HISTOGRAM_BUCKETS,TASKS_LAUNCH_PREPARATION_HISTOGRAM_METRICS,TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS,TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_METRICS,+ TASKS_NETWORK_BYTES_HISTOGRAM_BUCKETS,+ TASKS_NETWORK_BYTES_HISTOGRAM_METRICS,TASKS_RUN_TOKENS_HISTOGRAM_BUCKETS,TASKS_RUN_TOKENS_HISTOGRAM_METRICS,TASKS_RUN_TURNS_HISTOGRAM_BUCKETS,@@ -38,12 +42,16 @@ from products.tasks.backend.temporal.process_task.workflow import ProcessTaskWor__all__ = ["ACTIVITIES",+ "TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_BUCKETS",+ "TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_METRICS","TASKS_LATENCY_HISTOGRAM_BUCKETS","TASKS_LATENCY_HISTOGRAM_METRICS","TASKS_LAUNCH_PREPARATION_HISTOGRAM_BUCKETS","TASKS_LAUNCH_PREPARATION_HISTOGRAM_METRICS","TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS","TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_METRICS",+ "TASKS_NETWORK_BYTES_HISTOGRAM_BUCKETS",+ "TASKS_NETWORK_BYTES_HISTOGRAM_METRICS","TASKS_RUN_TOKENS_HISTOGRAM_BUCKETS","TASKS_RUN_TOKENS_HISTOGRAM_METRICS","TASKS_RUN_TURNS_HISTOGRAM_BUCKETS",diff --git a/products/tasks/backend/logic/services/modal_sandbox.py b/products/tasks/backend/logic/services/modal_sandbox.pyindex f8a619baaf61..c51e961d47e0 100644--- a/products/tasks/backend/logic/services/modal_sandbox.py+++ b/products/tasks/backend/logic/services/modal_sandbox.py@@ -25,6 +25,7 @@ from django.conf import settingsimport modalimport requestsfrom cachetools import TTLCache, cached+from modal._utils.async_utils import synchronizer as modal_synchronizerfrom modal.exception import (ConnectionError as ModalConnectionError,InvalidError as ModalInvalidError,@@ -32,6 +33,7 @@ from modal.exception import (ServiceError as ModalServiceError,TimeoutError as ModalTimeoutError,)+from modal_proto import api_pb2from python_socks import ProxyError as SocksProxyErrorfrom semantic_version import NpmSpec@@ -1998,6 +2000,26 @@ class ModalSandbox(AgentServerLaunchMixin):return "timed out"return f"exited with code {returncode}"+ def exit_status(self) -> str | None:+ try:+ returncode = self._sandbox.poll()+ except Exception as e:+ logger.warning(f"Failed to poll sandbox {self.id} for its exit status: {e}")+ return "unknown"+ if returncode is None:+ return "running"+ try:+ result = getattr(modal_synchronizer._translate_in(self._sandbox), "_result", None)+ if result is not None:+ return api_pb2.GenericResult.GenericStatus.Name(result.status).lower().removeprefix("generic_status_")+ except Exception as e:+ logger.warning(f"Failed to read exit status for sandbox {self.id}: {e}")+ if returncode == 124:+ return "timeout"+ if returncode == 137:+ return "terminated"+ return "success" if returncode == 0 else "failure"+@propertydef name(self) -> str:return self.config.namediff --git a/products/tasks/backend/logic/services/sandbox.py b/products/tasks/backend/logic/services/sandbox.pyindex 7633bdc93853..388ad19f7e82 100644--- a/products/tasks/backend/logic/services/sandbox.py+++ b/products/tasks/backend/logic/services/sandbox.py@@ -733,6 +733,9 @@ class SandboxBase(ABC):def exit_reason(self) -> str | None:return None+ def exit_status(self) -> str | None:+ return None+def read_agent_server_session_init_ms(self) -> int | None:return Nonediff --git a/products/tasks/backend/logic/services/sandbox_teardown_stats.py b/products/tasks/backend/logic/services/sandbox_teardown_stats.pynew file mode 100644index 000000000000..cfbf1f86adcc--- /dev/null+++ b/products/tasks/backend/logic/services/sandbox_teardown_stats.py@@ -0,0 +1,105 @@+from posthog.dataclasses import frozen++from products.tasks.backend.logic.services.memory_watchdog import build_memory_watchdog_read_command++SANDBOX_TEARDOWN_STATS_SEPARATOR = "__posthog_sandbox_teardown_stats__"+++@frozen+class SandboxTeardownStats:+ rx_bytes: int | None+ tx_bytes: int | None+ cpu_pressure_ratio: float | None+++def build_sandbox_teardown_stats_command() -> str:+ return (+ f"cat /proc/net/dev 2>/dev/null || true; echo {SANDBOX_TEARDOWN_STATS_SEPARATOR}; "+ f"cat /proc/pressure/cpu 2>/dev/null || true; echo {SANDBOX_TEARDOWN_STATS_SEPARATOR}; "+ "cat /proc/uptime 2>/dev/null || true"+ )+++def build_sandbox_teardown_read_command() -> str:+ return (+ f"{build_memory_watchdog_read_command()}; echo {SANDBOX_TEARDOWN_STATS_SEPARATOR}; "+ f"{build_sandbox_teardown_stats_command()}"+ )+++@frozen+class SandboxTeardownReadOutput:+ watchdog_output: str+ stats_output: str+++@frozen+class SandboxNetworkBytes:+ rx_bytes: int | None+ tx_bytes: int | None+++def split_sandbox_teardown_read_output(stdout: str) -> SandboxTeardownReadOutput:+ watchdog_output, _, stats_output = stdout.partition(SANDBOX_TEARDOWN_STATS_SEPARATOR)+ return SandboxTeardownReadOutput(watchdog_output=watchdog_output, stats_output=stats_output)+++def _parse_network_bytes(net_dev: str) -> SandboxNetworkBytes:+ rx_total = 0+ tx_total = 0+ found = False+ for line in net_dev.splitlines():+ if ":" not in line:+ continue+ interface, _, counters = line.partition(":")+ if interface.strip() == "lo":+ continue+ fields = counters.split()+ if len(fields) < 9:+ continue+ try:+ rx_bytes = int(fields[0])+ tx_bytes = int(fields[8])+ except ValueError:+ continue+ rx_total += rx_bytes+ tx_total += tx_bytes+ found = True+ if not found:+ return SandboxNetworkBytes(rx_bytes=None, tx_bytes=None)+ return SandboxNetworkBytes(rx_bytes=rx_total, tx_bytes=tx_total)+++def _parse_cpu_pressure_ratio(pressure: str, uptime: str) -> float | None:+ stall_usec: int | None = None+ for line in pressure.splitlines():+ parts = line.split()+ if not parts or parts[0] != "some":+ continue+ for part in parts[1:]:+ key, _, value = part.partition("=")+ if key == "total":+ try:+ stall_usec = int(value)+ except ValueError:+ return None+ if stall_usec is None:+ return None+ try:+ uptime_seconds = float(uptime.split()[0])+ except (IndexError, ValueError):+ return None+ if uptime_seconds <= 0:+ return None+ return min(max(stall_usec / (uptime_seconds * 1e6), 0.0), 1.0)+++def parse_sandbox_teardown_stats(output: str) -> SandboxTeardownStats:+ sections = output.split(SANDBOX_TEARDOWN_STATS_SEPARATOR)+ net_dev, pressure, uptime = [*sections, "", "", ""][:3]+ network = _parse_network_bytes(net_dev)+ return SandboxTeardownStats(+ rx_bytes=network.rx_bytes,+ tx_bytes=network.tx_bytes,+ cpu_pressure_ratio=_parse_cpu_pressure_ratio(pressure, uptime),+ )diff --git a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py b/products/tasks/backend/logic/services/tests/test_modal_sandbox.pyindex af4fec736ad9..ca3dc98c830b 100644--- a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py+++ b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py@@ -22,6 +22,7 @@ from modal.exception import (ServiceError as ModalServiceError,TimeoutError as ModalTimeoutError,)+from modal_proto import api_pb2from parameterized import parameterizedfrom python_socks import ProxyError as SocksProxyErrorfrom requests.exceptions import (@@ -1607,6 +1608,25 @@ class TestModalSandboxAgentServerStartupHelpers:sandbox._sandbox = MagicMock()return sandbox+ @pytest.mark.parametrize(+ "poll_result,status,expected",+ [+ (None, api_pb2.GenericResult.GENERIC_STATUS_UNSPECIFIED, "running"),+ (RuntimeError("provider unavailable"), api_pb2.GenericResult.GENERIC_STATUS_UNSPECIFIED, "unknown"),+ (137, api_pb2.GenericResult.GENERIC_STATUS_MEMORY_MANAGER_EVICTION, "memory_manager_eviction"),+ (0, api_pb2.GenericResult.GENERIC_STATUS_IDLE_TIMEOUT, "idle_timeout"),+ ],+ )+ def test_exit_status_reads_modal_generic_status(self, poll_result, status, expected):+ sandbox = self._make_sandbox()+ if isinstance(poll_result, Exception):+ sandbox._sandbox.poll.side_effect = poll_result+ else:+ sandbox._sandbox.poll.return_value = poll_result+ sandbox._sandbox._result.status = status++ assert sandbox.exit_status() == expected+def test_free_agent_server_port_terminates_existing_process(self):sandbox = self._make_sandbox()sandbox.execute = MagicMock(return_value=ExecutionResult(stdout="", stderr="", exit_code=0, error=None))diff --git a/products/tasks/backend/logic/services/tests/test_sandbox_teardown_stats.py b/products/tasks/backend/logic/services/tests/test_sandbox_teardown_stats.pynew file mode 100644index 000000000000..a2b785d67b4f--- /dev/null+++ b/products/tasks/backend/logic/services/tests/test_sandbox_teardown_stats.py@@ -0,0 +1,46 @@+import pytest++from products.tasks.backend.logic.services.sandbox_teardown_stats import (+ SANDBOX_TEARDOWN_STATS_SEPARATOR,+ SandboxTeardownStats,+ parse_sandbox_teardown_stats,+)++NET_DEV = """Inter-| Receive | Transmit+ face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed+ lo: 9999999 100 0 0 0 0 0 0 9999999 100 0 0 0 0 0 0+ eth0: 1000000 2000 0 0 0 0 0 0 300000 1000 0 0 0 0 0 0+ eth1: 500000 100 0 0 0 0 0 0 20000 50 0 0 0 0 0 0+"""+PRESSURE = "some avg10=0.00 avg60=0.00 avg300=0.00 total=25000000\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n"+UPTIME = "100.00 350.00\n"+++def _output(net_dev: str, pressure: str, uptime: str) -> str:+ return f"{net_dev}\n{SANDBOX_TEARDOWN_STATS_SEPARATOR}\n{pressure}\n{SANDBOX_TEARDOWN_STATS_SEPARATOR}\n{uptime}"+++ "output,expected",+ [+ (+ _output(NET_DEV, PRESSURE, UPTIME),+ SandboxTeardownStats(rx_bytes=1_500_000, tx_bytes=320_000, cpu_pressure_ratio=0.25),+ ),+ (+ _output(NET_DEV, "", UPTIME),+ SandboxTeardownStats(rx_bytes=1_500_000, tx_bytes=320_000, cpu_pressure_ratio=None),+ ),+ (+ _output(NET_DEV, "some avg10=0.00 total=500000000\n", UPTIME),+ SandboxTeardownStats(rx_bytes=1_500_000, tx_bytes=320_000, cpu_pressure_ratio=1.0),+ ),+ (+ "garbage: not numbers\nmore garbage",+ SandboxTeardownStats(rx_bytes=None, tx_bytes=None, cpu_pressure_ratio=None),+ ),+ ],+ ids=["full", "missing_psi", "clamped", "garbage"],+)+def test_parse_sandbox_teardown_stats(output: str, expected: SandboxTeardownStats) -> None:+ assert parse_sandbox_teardown_stats(output) == expecteddiff --git a/products/tasks/backend/temporal/metrics.py b/products/tasks/backend/temporal/metrics.pyindex d04dfd69282f..56d25d8cd822 100644--- a/products/tasks/backend/temporal/metrics.py+++ b/products/tasks/backend/temporal/metrics.py@@ -132,6 +132,21 @@ TASKS_RUN_TURNS_HISTOGRAM_BUCKETS = [TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_METRICS = ("tasks_sandbox_memory_peak_ratio",)TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1.0]+TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_METRICS = ("tasks_sandbox_cpu_pressure_ratio",)+TASKS_CPU_PRESSURE_RATIO_HISTOGRAM_BUCKETS = TASKS_MEMORY_PEAK_RATIO_HISTOGRAM_BUCKETS++TASKS_NETWORK_BYTES_HISTOGRAM_METRICS = ("tasks_sandbox_network_bytes",)+TASKS_NETWORK_BYTES_HISTOGRAM_BUCKETS = [+ 1_048_576.0,+ 4_194_304.0,+ 16_777_216.0,+ 67_108_864.0,+ 268_435_456.0,+ 1_073_741_824.0,+ 4_294_967_296.0,+ 17_179_869_184.0,+]+_RUN_TOKEN_KINDS = {"input": "input_tokens","output": "output_tokens",@@ -388,6 +403,36 @@ def record_memory_peak_ratio(ratio: float) -> None:pass+def increment_sandbox_exit_status(status: str, runtime: str) -> None:+ try:+ _metric_meter({"status": status, "runtime": runtime}).create_counter(+ "tasks_sandbox_exit_status",+ "Sandbox exit status read at cleanup, running when the sandbox was still alive, unknown when the poll failed",+ ).add(1)+ except Exception:+ pass+++def record_sandbox_network_bytes(direction: str, value: int) -> None:+ try:+ _metric_meter({"direction": direction}).create_histogram(+ "tasks_sandbox_network_bytes",+ "Sandbox network bytes over non-loopback interfaces, read at teardown",+ ).record(value)+ except Exception:+ pass+++def record_sandbox_cpu_pressure_ratio(ratio: float) -> None:+ try:+ _metric_meter().create_histogram_float(+ "tasks_sandbox_cpu_pressure_ratio",+ "Fraction of sandbox uptime with some task stalled on CPU, read at teardown",+ ).record(ratio)+ except Exception:+ pass++def increment_tool_call_only_heartbeat() -> None:try:_metric_meter().create_counter(diff --git a/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py b/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.pyindex 24c6f0ba422d..ed822d2d37bd 100644--- a/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py+++ b/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py@@ -20,6 +20,7 @@ from products.tasks.backend.logic.stream.redis_stream import ()from products.tasks.backend.models import SandboxSession, TaskRunfrom products.tasks.backend.redis import run_uses_dedicated_stream+from products.tasks.backend.temporal.metrics import increment_sandbox_exit_status, sandbox_runtime_labelfrom products.tasks.backend.temporal.observability import log_activity_executionlogger = logging.getLogger(__name__)@@ -93,6 +94,17 @@ def cleanup_sandbox_now(input: CleanupSandboxInput) -> None:exc_info=True,)+ try:+ exit_status = sandbox.exit_status()+ if exit_status is not None:+ session = (+ SandboxSession.objects.unscoped().filter(sandbox_id=input.sandbox_id).only("vm_runtime").first()+ )+ runtime = sandbox_runtime_label(session.vm_runtime) if session is not None else "unknown"+ increment_sandbox_exit_status(exit_status, runtime)+ except Exception:+ logger.warning("cleanup_sandbox_exit_status_failed", extra={"sandbox_id": input.sandbox_id}, exc_info=True)+cpu_usage_usec, cpu_usage_measured_at = measure_sandbox_cpu_usage(sandbox)billed_cpu_usage_usec = measure_sandbox_billed_cpu_usage(sandbox)diff --git a/products/tasks/backend/temporal/process_task/activities/read_sandbox_logs.py b/products/tasks/backend/temporal/process_task/activities/read_sandbox_logs.pyindex 9bc88b61c5d5..3788cf1bf8c2 100644--- a/products/tasks/backend/temporal/process_task/activities/read_sandbox_logs.py+++ b/products/tasks/backend/temporal/process_task/activities/read_sandbox_logs.py@@ -8,15 +8,19 @@ from temporalio import activityfrom posthog.temporal.common.utils import asyncifyfrom products.tasks.backend.exceptions import SandboxNotRunningError-from products.tasks.backend.logic.services.memory_watchdog import (- build_memory_watchdog_read_command,- parse_memory_watchdog_read_output,-)+from products.tasks.backend.logic.services.memory_watchdog import parse_memory_watchdog_read_outputfrom products.tasks.backend.logic.services.sandbox import SandboxBase, get_sandbox_class_for_sandbox_id+from products.tasks.backend.logic.services.sandbox_teardown_stats import (+ build_sandbox_teardown_read_command,+ parse_sandbox_teardown_stats,+ split_sandbox_teardown_read_output,+)from products.tasks.backend.temporal.metrics import (increment_memory_watchdog_events,increment_memory_watchdog_teardown,record_memory_peak_ratio,+ record_sandbox_cpu_pressure_ratio,+ record_sandbox_network_bytes,)from products.tasks.backend.temporal.observability import log_activity_execution@@ -34,9 +38,8 @@ class ReadSandboxLogsInput:SANDBOX_TERMINATED_MESSAGE = "Sandbox terminated before logs could be captured; no agent-server logs available."-def _report_memory_watchdog(sandbox: SandboxBase, sandbox_id: str, run_id: str | None) -> None:- result = sandbox.execute(build_memory_watchdog_read_command(), timeout_seconds=10)- summary = parse_memory_watchdog_read_output(result.stdout, time.time())+def _report_memory_watchdog(stdout: str, sandbox_id: str, run_id: str | None) -> None:+ summary = parse_memory_watchdog_read_output(stdout, time.time())logger.info(f"Sandbox memory watchdog summary run_id={run_id} sandbox_id={sandbox_id} kills={summary.kill_count} "f"no_target={summary.no_target} pressure_episodes={summary.pressure_episodes} "@@ -50,6 +53,27 @@ def _report_memory_watchdog(sandbox: SandboxBase, sandbox_id: str, run_id: str |record_memory_peak_ratio(summary.peak_ratio)+def _report_teardown_stats(stdout: str, sandbox_id: str, run_id: str | None) -> None:+ stats = parse_sandbox_teardown_stats(stdout)+ logger.info(+ f"Sandbox teardown stats run_id={run_id} sandbox_id={sandbox_id} rx_bytes={stats.rx_bytes} "+ f"tx_bytes={stats.tx_bytes} cpu_pressure_ratio={stats.cpu_pressure_ratio}"+ )+ if stats.rx_bytes is not None:+ record_sandbox_network_bytes("rx", stats.rx_bytes)+ if stats.tx_bytes is not None:+ record_sandbox_network_bytes("tx", stats.tx_bytes)+ if stats.cpu_pressure_ratio is not None:+ record_sandbox_cpu_pressure_ratio(stats.cpu_pressure_ratio)+++def _report_teardown(sandbox: SandboxBase, sandbox_id: str, run_id: str | None) -> None:+ result = sandbox.execute(build_sandbox_teardown_read_command(), timeout_seconds=10)+ output = split_sandbox_teardown_read_output(result.stdout)+ _report_memory_watchdog(output.watchdog_output, sandbox_id, run_id)+ _report_teardown_stats(output.stats_output, sandbox_id, run_id)++@activity.defn@asyncifydef read_sandbox_logs(input: ReadSandboxLogsInput) -> str:@@ -108,9 +132,9 @@ def read_sandbox_logs(input: ReadSandboxLogsInput) -> str:logger.debug("agentsh audit query failed for sandbox %s", input.sandbox_id, exc_info=True)try:- _report_memory_watchdog(sandbox, input.sandbox_id, input.run_id)+ _report_teardown(sandbox, input.sandbox_id, input.run_id)except Exception:- logger.warning("Memory watchdog summary failed for sandbox %s", input.sandbox_id, exc_info=True)+ logger.warning("Teardown stats failed for sandbox %s", input.sandbox_id, exc_info=True)return logsexcept SandboxNotRunningError:diff --git a/products/tasks/backend/temporal/process_task/tests/test_read_sandbox_logs.py b/products/tasks/backend/temporal/process_task/tests/test_read_sandbox_logs.pyindex bbd0133ace79..e45e145ebed8 100644--- a/products/tasks/backend/temporal/process_task/tests/test_read_sandbox_logs.py+++ b/products/tasks/backend/temporal/process_task/tests/test_read_sandbox_logs.py@@ -7,8 +7,8 @@ from products.tasks.backend.logic.services.memory_watchdog import (MEMORY_WATCHDOG_EVENTS_SEPARATOR,MEMORY_WATCHDOG_NOW_PREFIX,MEMORY_WATCHDOG_STATE_SEPARATOR,- build_memory_watchdog_read_command,)+from products.tasks.backend.logic.services.sandbox_teardown_stats import build_sandbox_teardown_read_commandfrom products.tasks.backend.temporal.process_task.activities.read_sandbox_logs import (SANDBOX_TERMINATED_MESSAGE,ReadSandboxLogsInput,@@ -71,7 +71,7 @@ def test_reports_memory_watchdog_kills_to_metrics():sandbox = MagicMock()sandbox.is_running.return_value = Truesandbox.execute.side_effect = lambda command, **_: MagicMock(- stdout=watchdog_output if command == build_memory_watchdog_read_command() else ""+ stdout=watchdog_output if command == build_sandbox_teardown_read_command() else "")module = "products.tasks.backend.temporal.process_task.activities.read_sandbox_logs"diff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.pyindex 1b2a81b7a548..93a128581d8e 100644--- a/products/tasks/backend/tests/test_api.py+++ b/products/tasks/backend/tests/test_api.py@@ -11624,6 +11624,7 @@ class TestTaskRunCancelAPI(BaseTaskAPITest):sandbox.method_calls,[call.stop_agent_server(),+ call.exit_status(),call.read_cpu_usage_usec(),call.read_billed_cpu_usage_usec(),call.destroy(),@@ -11668,6 +11669,7 @@ class TestTaskRunCancelAPI(BaseTaskAPITest):sandbox.method_calls,[call.stop_agent_server(),+ call.exit_status(),call.read_cpu_usage_usec(),call.read_billed_cpu_usage_usec(),call.destroy(),@@ -11721,6 +11723,7 @@ class TestTaskRunCancelAPI(BaseTaskAPITest):sandbox.method_calls,[call.stop_agent_server(),+ call.exit_status(),call.read_cpu_usage_usec(),call.read_billed_cpu_usage_usec(),call.destroy(),