Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(tasks): combine calls to modal (#120439)

Jonathan Mielood4c78756a516
diff --git a/products/tasks/backend/logic/services/agent_server_launcher.py b/products/tasks/backend/logic/services/agent_server_launcher.py
index 8510573fbf64..ee0c41984374 100644
--- a/products/tasks/backend/logic/services/agent_server_launcher.py
+++ b/products/tasks/backend/logic/services/agent_server_launcher.py
@@ -50,6 +50,7 @@ from products.tasks.backend.logic.services.sandbox import (
CLAUDE_CREDENTIAL_UNAVAILABLE_MESSAGE,
CODEX_CREDENTIAL_UNAVAILABLE_MESSAGE,
WORKING_DIR,
+ ExecutionResult,
SandboxBase,
build_agent_runtime_env_prefix,
build_agent_server_capability_probe,
@@ -213,6 +214,7 @@ class AgentServerPreflight:
reused: bool
capabilities: frozenset[str]
memory_watchdog_missing: bool = False
+ launch_prepared: bool = False
def supports(self, capability: str) -> bool:
return capability in self.capabilities
@@ -221,6 +223,16 @@ class AgentServerPreflight:
def build_agent_server_preflight_script(
*, probe_health: bool, executable_paths: tuple[str, ...], probe_memory_watchdog: bool = False
) -> str:
+ lines = build_agent_server_preflight_lines(
+ probe_health=probe_health, executable_paths=executable_paths, probe_memory_watchdog=probe_memory_watchdog
+ )
+ lines.append("exit 0")
+ return "\n".join(lines)
+
+
+def build_agent_server_preflight_lines(
+ *, probe_health: bool, executable_paths: tuple[str, ...], probe_memory_watchdog: bool = False
+) -> list[str]:
lines = [
f"if ! chmod +x {shlex.quote(path)}; then exit {AGENT_SERVER_PREFLIGHT_CHMOD_EXIT_CODE}; fi"
for path in executable_paths
@@ -246,8 +258,7 @@ def build_agent_server_preflight_script(
AGENT_SERVER_FREE_PORT_SCRIPT,
]
)
- lines.append("exit 0")
- return "\n".join(lines)
+ return lines
CLAUDE_CREDENTIAL_UNAVAILABLE_MARKER = "claude_credential_unavailable"
@@ -556,9 +567,6 @@ class AgentServerLaunchMixin(SandboxBase):
def _agent_server_reuse_enabled(self) -> bool:
return True
- def _on_agent_server_reused(self) -> None:
- return None
-
def _agent_server_preflight(self, allowed_domains: list[str] | None) -> AgentServerPreflight:
executable_paths = self._install_agent_server_launch_files()
result = self.execute_in_task_filesystem(
@@ -574,6 +582,11 @@ class AgentServerLaunchMixin(SandboxBase):
),
timeout_seconds=AGENT_SERVER_PREFLIGHT_TIMEOUT_SECONDS,
)
+ return self._read_agent_server_preflight(result, executable_paths, allowed_domains)
+
+ def _read_agent_server_preflight(
+ self, result: ExecutionResult, executable_paths: tuple[str, ...], allowed_domains: list[str] | None
+ ) -> AgentServerPreflight:
if result.exit_code == AGENT_SERVER_PREFLIGHT_CHMOD_EXIT_CODE:
raise SandboxExecutionError(
"Failed to set permissions on required sandbox file",
@@ -603,7 +616,6 @@ class AgentServerLaunchMixin(SandboxBase):
if AGENT_SERVER_PREFLIGHT_REUSE_MARKER in lines:
if allowed_domains is None or self._agentsh_daemon_is_healthy():
logger.info(f"Agent-server already healthy in sandbox {self.id}; skipping relaunch")
- self._on_agent_server_reused()
return AgentServerPreflight(reused=True, capabilities=capabilities)
self._free_agent_server_port()
return AgentServerPreflight(
@@ -729,7 +741,8 @@ class AgentServerLaunchMixin(SandboxBase):
org, repo = repository.lower().split("/")
repo_path = f"/tmp/workspace/repos/{org}/{repo}"
- self._prepare_agent_server_launch(allowed_domains)
+ if not preflight.launch_prepared:
+ self._prepare_agent_server_launch(allowed_domains)
memory_watchdog_ready = self._install_memory_watchdog(preflight)
codex_run_token_file = CODEX_RUN_TOKEN_FILE if codex_run_token else None
diff --git a/products/tasks/backend/logic/services/modal_launch_preparation.py b/products/tasks/backend/logic/services/modal_launch_preparation.py
index 51b1495b123f..fc5dbd73f598 100644
--- a/products/tasks/backend/logic/services/modal_launch_preparation.py
+++ b/products/tasks/backend/logic/services/modal_launch_preparation.py
@@ -1,6 +1,8 @@
import shlex
import base64
+from posthog.dataclasses import frozen
+
from products.tasks.backend.constants import DEFAULT_SANDBOX_WORKING_DIR
from .agentsh import (
@@ -15,25 +17,78 @@ from .agentsh import (
generate_policy_yaml,
read_gh_guard_script,
)
-from .task_filesystem import TASK_FILESYSTEM_SCRIPT, read_task_filesystem_script
+from .task_filesystem import TASK_FILESYSTEM_SCRIPT, read_task_filesystem_script, task_filesystem_command
+
+
+@frozen
+class LaunchFile:
+ path: str
+ payload: bytes
+ mode: str
def build_modal_launch_preparation_script(allowed_domains: list[str] | None) -> str:
- files = [
- (BASH_ENV_SCRIPT, generate_bash_env_script().encode(), "644"),
- (GH_GUARD_INSTALL_PATH, read_gh_guard_script(), "755"),
- (TASK_FILESYSTEM_SCRIPT, read_task_filesystem_script(), "644"),
- ]
+ files = _base_files()
if allowed_domains is not None:
- files.extend(
- [
- ("/etc/agentsh/config.yaml", generate_config_yaml(enable_ptrace=True, full_trace=True).encode(), "644"),
- ("/etc/agentsh/policies/default.yaml", generate_policy_yaml(allowed_domains).encode(), "644"),
- (ENV_WRAPPER_SCRIPT, generate_env_wrapper().encode(), "755"),
- ]
- )
+ files.extend(_agentsh_files(allowed_domains))
+ script = _script_header()
+ script.extend(_install_lines(files))
+ script.extend(_installed_lines())
+ if allowed_domains is not None:
+ script.extend(_daemon_session_lines())
+ return "\n".join(script) + "\n"
+
+
+def build_modal_agent_server_launch_script(
+ allowed_domains: list[str] | None, *, bundled_skills_clear_command: str, preflight_lines: list[str]
+) -> str:
+ script = _script_header()
+ script.extend(_install_lines(_base_files()))
+ script.extend(_installed_lines())
+ script.extend(
+ [
+ "stage=bundled_skills",
+ f"(umask 022 && {task_filesystem_command(bundled_skills_clear_command)})",
+ "stage=preflight",
+ "set +euo pipefail",
+ *preflight_lines,
+ "set -euo pipefail",
+ ]
+ )
+ if allowed_domains is not None:
+ script.append("stage=install")
+ script.extend(_install_lines(_agentsh_files(allowed_domains)))
+ script.append("installed_at=$(date +%s%3N)")
+ script.extend(_daemon_session_lines())
+ return "\n".join(script) + "\n"
+
+
+def _base_files() -> list[LaunchFile]:
+ return [
+ LaunchFile(path=BASH_ENV_SCRIPT, payload=generate_bash_env_script().encode(), mode="644"),
+ LaunchFile(path=GH_GUARD_INSTALL_PATH, payload=read_gh_guard_script(), mode="755"),
+ LaunchFile(path=TASK_FILESYSTEM_SCRIPT, payload=read_task_filesystem_script(), mode="644"),
+ ]
+
- script = [
+def _agentsh_files(allowed_domains: list[str]) -> list[LaunchFile]:
+ return [
+ LaunchFile(
+ path="/etc/agentsh/config.yaml",
+ payload=generate_config_yaml(enable_ptrace=True, full_trace=True).encode(),
+ mode="644",
+ ),
+ LaunchFile(
+ path="/etc/agentsh/policies/default.yaml",
+ payload=generate_policy_yaml(allowed_domains).encode(),
+ mode="644",
+ ),
+ LaunchFile(path=ENV_WRAPPER_SCRIPT, payload=generate_env_wrapper().encode(), mode="755"),
+ ]
+
+
+def _script_header() -> list[str]:
+ return [
"#!/bin/bash",
"set -euo pipefail",
"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
@@ -60,59 +115,63 @@ def build_modal_launch_preparation_script(allowed_domains: list[str] | None) ->
"}",
"trap cleanup EXIT",
]
- for path, payload, mode in files:
- destination = shlex.quote(path)
- encoded = shlex.quote(base64.b64encode(payload).decode())
- script.extend(
+
+
+def _install_lines(files: list[LaunchFile]) -> list[str]:
+ lines: list[str] = ["first_staged=${#staged_files[@]}"]
+ for file in files:
+ destination = shlex.quote(file.path)
+ encoded = shlex.quote(base64.b64encode(file.payload).decode())
+ lines.extend(
[
f'mkdir -p -- "$(dirname -- {destination})"',
- f"temporary=$(mktemp {shlex.quote(path + '.tmp.XXXXXX')})",
+ f"temporary=$(mktemp {shlex.quote(file.path + '.tmp.XXXXXX')})",
'staged_files+=("$temporary")',
f'printf %s {encoded} | base64 -d > "$temporary"',
- f'chmod {mode} "$temporary"',
+ f'chmod {file.mode} "$temporary"',
]
)
- for index, (path, _, _) in enumerate(files):
- script.append(f"test ! -d {shlex.quote(path)}")
- script.append(f'mv -f -- "${{staged_files[{index}]}}" {shlex.quote(path)}')
- script.extend(
- [
- "installed_at=$(date +%s%3N)",
- 'printf "__posthog_launch_preparation_install_ms=%s\\n" "$((installed_at - started_at))"',
- ]
- )
+ for index, file in enumerate(files):
+ lines.append(f"test ! -d {shlex.quote(file.path)}")
+ lines.append(f'mv -f -- "${{staged_files[$((first_staged + {index}))]}}" {shlex.quote(file.path)}')
+ return lines
- if allowed_domains is not None:
- script.extend(
- [
- "stage=daemon_session",
- "pkill -f '[a]gentsh server' || [ $? -eq 1 ]",
- "for i in $(seq 1 50); do",
- " if ! pgrep -f '[a]gentsh server' > /dev/null; then break; fi",
- " sleep 0.1",
- "done",
- "if pgrep -f '[a]gentsh server' > /dev/null; then exit 1; fi",
- "mkdir -p /var/log/agentsh /var/lib/agentsh/sessions",
- f"rm -f -- {shlex.quote(SESSION_ID_FILE)}",
- f"nohup /usr/bin/python3 -I {TASK_FILESYSTEM_SCRIPT} enter -- /usr/bin/agentsh server --config /etc/agentsh/config.yaml > /var/log/agentsh/agentsh.log 2>&1 &",
- "daemon_pid=$!",
- "healthy=0",
- "for i in $(seq 1 30); do",
- f" if [ \"$(curl -s --max-time 1 -o /dev/null -w '%{{http_code}}' http://127.0.0.1:{AGENTSH_DAEMON_PORT}/health || true)\" = 200 ]; then",
- " healthy=1",
- " break",
- " fi",
- " sleep 0.5",
- "done",
- '[ "$healthy" -eq 1 ]',
- 'kill -0 "$daemon_pid"',
- f"session_id=$(timeout 60s /usr/bin/agentsh session create --workspace {shlex.quote(DEFAULT_SANDBOX_WORKING_DIR)} --policy default --json | jq -er '.id | select(type == \"string\" and length > 0)')",
- f"temporary=$(mktemp {shlex.quote(SESSION_ID_FILE + '.tmp.XXXXXX')})",
- 'staged_files+=("$temporary")',
- 'printf %s "$session_id" > "$temporary"',
- f'mv -f -- "$temporary" {shlex.quote(SESSION_ID_FILE)}',
- f"test -s {shlex.quote(SESSION_ID_FILE)}",
- 'printf "__posthog_launch_preparation_daemon_session_ms=%s\\n" "$(($(date +%s%3N) - installed_at))"',
- ]
- )
- return "\n".join(script) + "\n"
+
+def _installed_lines() -> list[str]:
+ return [
+ "installed_at=$(date +%s%3N)",
+ 'printf "__posthog_launch_preparation_install_ms=%s\\n" "$((installed_at - started_at))"',
+ ]
+
+
+def _daemon_session_lines() -> list[str]:
+ return [
+ "stage=daemon_session",
+ "pkill -f '[a]gentsh server' || [ $? -eq 1 ]",
+ "for i in $(seq 1 50); do",
+ " if ! pgrep -f '[a]gentsh server' > /dev/null; then break; fi",
+ " sleep 0.1",
+ "done",
+ "if pgrep -f '[a]gentsh server' > /dev/null; then exit 1; fi",
+ "mkdir -p /var/log/agentsh /var/lib/agentsh/sessions",
+ f"rm -f -- {shlex.quote(SESSION_ID_FILE)}",
+ f"nohup /usr/bin/python3 -I {TASK_FILESYSTEM_SCRIPT} enter -- /usr/bin/agentsh server --config /etc/agentsh/config.yaml > /var/log/agentsh/agentsh.log 2>&1 &",
+ "daemon_pid=$!",
+ "healthy=0",
+ "for i in $(seq 1 30); do",
+ f" if [ \"$(curl -s --max-time 1 -o /dev/null -w '%{{http_code}}' http://127.0.0.1:{AGENTSH_DAEMON_PORT}/health || true)\" = 200 ]; then",
+ " healthy=1",
+ " break",
+ " fi",
+ " sleep 0.5",
+ "done",
+ '[ "$healthy" -eq 1 ]',
+ 'kill -0 "$daemon_pid"',
+ f"session_id=$(timeout 60s /usr/bin/agentsh session create --workspace {shlex.quote(DEFAULT_SANDBOX_WORKING_DIR)} --policy default --json | jq -er '.id | select(type == \"string\" and length > 0)')",
+ f"temporary=$(mktemp {shlex.quote(SESSION_ID_FILE + '.tmp.XXXXXX')})",
+ 'staged_files+=("$temporary")',
+ 'printf %s "$session_id" > "$temporary"',
+ f'mv -f -- "$temporary" {shlex.quote(SESSION_ID_FILE)}',
+ f"test -s {shlex.quote(SESSION_ID_FILE)}",
+ 'printf "__posthog_launch_preparation_daemon_session_ms=%s\\n" "$(($(date +%s%3N) - installed_at))"',
+ ]
diff --git a/products/tasks/backend/logic/services/modal_sandbox.py b/products/tasks/backend/logic/services/modal_sandbox.py
index a2d86a238fb2..f8a619baaf61 100644
--- a/products/tasks/backend/logic/services/modal_sandbox.py
+++ b/products/tasks/backend/logic/services/modal_sandbox.py
@@ -11,7 +11,8 @@ import asyncio
import logging
import tempfile
import threading
-from collections.abc import Iterable
+import dataclasses
+from collections.abc import Callable, Iterable
from dataclasses import dataclass
from functools import lru_cache
from http import HTTPStatus
@@ -60,12 +61,6 @@ from products.tasks.backend.exceptions import (
SnapshotFileLimitExceededError,
SnapshotTimeoutError,
)
-from products.tasks.backend.logic.services.agentsh import (
- BASH_ENV_SCRIPT,
- GH_GUARD_INSTALL_PATH,
- generate_bash_env_script,
- read_gh_guard_script,
-)
from products.tasks.backend.logic.services.cpu_billing import (
CPU_BILLING_SAMPLER_PATH as CPU_BILLING_SAMPLER_PATH,
CPU_BILLING_STATE_PATH,
@@ -84,13 +79,16 @@ from products.tasks.backend.logic.services.local_skills import (
LocalSkillsCache,
populate_skills_directory,
)
-from products.tasks.backend.logic.services.modal_launch_preparation import build_modal_launch_preparation_script
+from products.tasks.backend.logic.services.modal_launch_preparation import (
+ build_modal_agent_server_launch_script,
+ build_modal_launch_preparation_script,
+)
from products.tasks.backend.logic.services.modal_provision_diagnostics import (
SandboxProvisionDiagnostics,
capture_modal_output_if_debug,
summarize_modal_output,
)
-from products.tasks.backend.logic.services.sandbox import redact_sandbox_command
+from products.tasks.backend.logic.services.sandbox import build_bundled_skills_clear_command, redact_sandbox_command
from products.tasks.backend.logic.services.task_filesystem import (
TASK_FILESYSTEM_SCRIPT,
TASK_FILESYSTEM_VERSION,
@@ -101,8 +99,13 @@ from products.tasks.backend.models import DevStackImageTarget, SandboxSnapshot
from .agent_server_launcher import (
AGENT_SERVER_PORT as AGENT_SERVER_PORT,
+ AGENT_SERVER_PREFLIGHT_CREDENTIAL_EXIT_CODE,
+ AGENT_SERVER_PREFLIGHT_REUSE_MARKER,
+ AGENT_SERVER_PREFLIGHT_TIMEOUT_SECONDS,
AgentServerLaunchMixin,
+ AgentServerPreflight,
_session_init_probe_hosts as _session_init_probe_hosts,
+ build_agent_server_preflight_lines,
)
from .sandbox import (
AgentServerResult,
@@ -185,6 +188,17 @@ _EMBEDDED_PROXY_STATUS = re.compile(r"\b([45]\d{2})\b")
ControlPlaneFailure = Literal["rate_limited", "unavailable"]
RUNNING_STATUS_CACHE_SECONDS = 10.0
+MODAL_LAUNCH_PREPARATION_TIMEOUT_SECONDS = 120
+MODAL_INLINE_COMMAND_MAX_BYTES = 60_000
+
+
+def _launch_preparation_failed_stage(stderr: str) -> str:
+ stage = re.search(
+ r"^__posthog_launch_preparation_failed=(install|bundled_skills|preflight|daemon_session)$",
+ stderr,
+ re.MULTILINE,
+ )
+ return stage.group(1) if stage else "unknown"
def _classify_control_plane_failure(error: BaseException) -> ControlPlaneFailure | None:
@@ -829,8 +843,9 @@ class ModalSandbox(AgentServerLaunchMixin):
self.provision_diagnostics = None
self._destroyed = False
- def _install_agent_server_launch_files(self) -> tuple[str, ...]:
- return ()
+ def _validate_agent_server_launch(self) -> None:
+ if not self._is_running_cached():
+ raise RuntimeError("Sandbox not in running state.")
def _wrap_agent_server_command(self, command: str) -> str:
return f"/usr/bin/python3 -I {TASK_FILESYSTEM_SCRIPT} enter -- {command}"
@@ -865,28 +880,72 @@ class ModalSandbox(AgentServerLaunchMixin):
def _sandbox_runtime(self) -> str | None:
return "vm" if self.config.is_vm else "gvisor"
- def _on_agent_server_reused(self) -> None:
- # A restored snapshot can carry a healthy agent-server with a stale bash-env
- # or gh shim from the snapshot's epoch. Refresh both before accepting reuse;
- # agentsh setup and session replacement stay on the fresh-launch path so
- # reuse doesn't disrupt the running server or its agentsh session.
- self._write_required_file(BASH_ENV_SCRIPT, generate_bash_env_script().encode())
- self._write_required_file(GH_GUARD_INSTALL_PATH, read_gh_guard_script())
- self._chmod_required(GH_GUARD_INSTALL_PATH, "+x")
+ def _agent_server_preflight(self, allowed_domains: list[str] | None) -> AgentServerPreflight:
+ script = build_modal_agent_server_launch_script(
+ allowed_domains,
+ bundled_skills_clear_command=build_bundled_skills_clear_command(),
+ preflight_lines=build_agent_server_preflight_lines(
+ probe_health=self._agent_server_reuse_enabled(),
+ executable_paths=(),
+ probe_memory_watchdog=settings.TASKS_SANDBOX_MEMORY_WATCHDOG_ENABLED,
+ ),
+ )
+ timeout_seconds = MODAL_LAUNCH_PREPARATION_TIMEOUT_SECONDS + AGENT_SERVER_PREFLIGHT_TIMEOUT_SECONDS
+ encoded_script = base64.b64encode(script.encode()).decode()
+ command = (
+ "script_path=$(mktemp /tmp/posthog-launch-preparation-XXXXXXXX) && "
+ f'printf %s {encoded_script} | base64 -d > "$script_path" && bash "$script_path"'
+ )
+
+ def run() -> ExecutionResult:
+ if len(command.encode()) <= MODAL_INLINE_COMMAND_MAX_BYTES:
+ return self.execute(command, timeout_seconds=timeout_seconds)
+ script_path = f"/tmp/posthog-launch-preparation-{uuid.uuid4().hex}.sh"
+ self._write_required_file(script_path, script.encode())
+ return self.execute(f"bash {shlex.quote(script_path)}", timeout_seconds=timeout_seconds)
+
+ result, total_ms = self._run_launch_preparation(
+ run, completed_exit_codes=(0, AGENT_SERVER_PREFLIGHT_CREDENTIAL_EXIT_CODE)
+ )
+ if _launch_preparation_failed_stage(result.stderr) == "bundled_skills":
+ raise RuntimeError(f"Failed to clear bundled skills in sandbox {self.id}: {result.stderr}")
+ if result.exit_code != AGENT_SERVER_PREFLIGHT_CREDENTIAL_EXIT_CODE:
+ self._finish_launch_preparation(result, upload_ms=0, total_ms=total_ms)
+ preflight = self._read_agent_server_preflight(result, (), allowed_domains)
+ if AGENT_SERVER_PREFLIGHT_REUSE_MARKER in result.stdout.splitlines():
+ return preflight
+ return dataclasses.replace(preflight, launch_prepared=True)
def _prepare_agent_server_launch(self, allowed_domains: list[str] | None) -> None:
script_path = f"/tmp/posthog-launch-preparation-{uuid.uuid4().hex}.sh"
+ upload_ms = 0
+
+ def upload_and_run() -> ExecutionResult:
+ nonlocal upload_ms
+ started_at = time.monotonic()
+ self._write_required_file(script_path, build_modal_launch_preparation_script(allowed_domains).encode())
+ upload_ms = round((time.monotonic() - started_at) * 1000)
+ return self.execute(
+ f"bash {shlex.quote(script_path)}", timeout_seconds=MODAL_LAUNCH_PREPARATION_TIMEOUT_SECONDS
+ )
+
+ result, total_ms = self._run_launch_preparation(upload_and_run, completed_exit_codes=(0,))
+ self._finish_launch_preparation(result, upload_ms=upload_ms, total_ms=total_ms)
+
+ def _run_launch_preparation(
+ self, run: Callable[[], ExecutionResult], *, completed_exit_codes: tuple[int, ...]
+ ) -> tuple[ExecutionResult, int]:
started_at = time.monotonic()
result: ExecutionResult | None = None
try:
- self._write_required_file(script_path, build_modal_launch_preparation_script(allowed_domains).encode())
- uploaded_at = time.monotonic()
- result = self.execute(f"bash {shlex.quote(script_path)}", timeout_seconds=120)
+ result = run()
finally:
total_ms = round((time.monotonic() - started_at) * 1000)
- record_launch_preparation_ms(
- total_ms, "COMPLETED" if result is not None and result.exit_code == 0 else "FAILED"
- )
+ completed = result is not None and result.exit_code in completed_exit_codes
+ record_launch_preparation_ms(total_ms, "COMPLETED" if completed else "FAILED")
+ return result, total_ms
+
+ def _finish_launch_preparation(self, result: ExecutionResult, *, upload_ms: int, total_ms: int) -> None:
timings = {
stage: int(duration)
for stage, duration in re.findall(
@@ -897,17 +956,14 @@ class ModalSandbox(AgentServerLaunchMixin):
"Modal launch preparation finished in sandbox %s: upload_ms=%d install_ms=%s "
"daemon_session_ms=%s total_ms=%d exit_code=%d",
self.id,
- round((uploaded_at - started_at) * 1000),
+ upload_ms,
timings.get("install_ms"),
timings.get("daemon_session_ms"),
total_ms,
result.exit_code,
)
if result.exit_code != 0:
- stage = re.search(
- r"^__posthog_launch_preparation_failed=(install|daemon_session)$", result.stderr, re.MULTILINE
- )
- preparation_stage = stage.group(1) if stage else "unknown"
+ preparation_stage = _launch_preparation_failed_stage(result.stderr)
agentsh_log = ""
if preparation_stage == "daemon_session":
try:
diff --git a/products/tasks/backend/logic/services/tests/test_modal_launch_preparation.py b/products/tasks/backend/logic/services/tests/test_modal_launch_preparation.py
index b9796c5f5c5b..f1117921de13 100644
--- a/products/tasks/backend/logic/services/tests/test_modal_launch_preparation.py
+++ b/products/tasks/backend/logic/services/tests/test_modal_launch_preparation.py
@@ -2,6 +2,7 @@ import os
import re
import shlex
import signal
+import itertools
import subprocess
from datetime import timedelta
from pathlib import Path
@@ -10,7 +11,11 @@ import pytest
from unittest.mock import MagicMock, patch
from products.tasks.backend.exceptions import SandboxExecutionError
-from products.tasks.backend.logic.services.agent_server_launcher import AgentServerPreflight
+from products.tasks.backend.logic.services.agent_server_launcher import (
+ AGENT_SERVER_PREFLIGHT_REUSE_MARKER,
+ AgentServerPreflight,
+ build_agent_server_preflight_lines,
+)
from products.tasks.backend.logic.services.agentsh import (
BASH_ENV_SCRIPT,
ENV_WRAPPER_SCRIPT,
@@ -26,13 +31,17 @@ from products.tasks.backend.logic.services.launch_preparation_metrics import (
launch_preparation_metric_context,
record_launch_preparation_ms,
)
-from products.tasks.backend.logic.services.modal_launch_preparation import build_modal_launch_preparation_script
+from products.tasks.backend.logic.services.modal_launch_preparation import (
+ build_modal_agent_server_launch_script,
+ build_modal_launch_preparation_script,
+)
from products.tasks.backend.logic.services.modal_sandbox import ModalSandbox
from products.tasks.backend.logic.services.sandbox import (
ExecutionResult,
SandboxConfig,
build_bundled_skills_clear_command,
)
+from products.tasks.backend.logic.services.task_filesystem import TASK_FILESYSTEM_SCRIPT
@pytest.fixture
@@ -69,9 +78,9 @@ esac
def run_preparation(
- root: Path, allowed_domains: list[str] | None, environment: dict[str, str]
+ root: Path, allowed_domains: list[str] | None, environment: dict[str, str], script: str | None = None
) -> subprocess.CompletedProcess[str]:
- script = build_modal_launch_preparation_script(allowed_domains)
+ script = script if script is not None else build_modal_launch_preparation_script(allowed_domains)
script = re.sub(r"/(?:tmp|etc|opt|var)/", lambda match: f"{root}{match.group()}", script)
script = script.replace(
"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
@@ -132,6 +141,39 @@ def test_preparation_installs_complete_files_and_can_be_reapplied(
assert not (tmp_path / SESSION_ID_FILE.lstrip("/")).exists()
+def run_launch_script(
+ root: Path, environment: dict[str, str], *, agent_server_healthy: bool
+) -> subprocess.CompletedProcess[str]:
+ health_body = '{"status":"ok","hasSession":true}' if agent_server_healthy else '{"status":"ok","hasSession":false}'
+ (root / "bin/curl").write_text(
+ f'#!/bin/bash\ncase "$*" in *localhost:8080/health*) echo {shlex.quote(health_body)} ;; *) echo 200 ;; esac\n'
+ )
+ script = build_modal_agent_server_launch_script(
+ ["example.com"],
+ bundled_skills_clear_command="true",
+ preflight_lines=build_agent_server_preflight_lines(probe_health=True, executable_paths=()),
+ ).replace(f"/usr/bin/python3 -I {TASK_FILESYSTEM_SCRIPT} enter -- ", "")
+ return run_preparation(root, ["example.com"], environment, script)
+
+
[email protected]("agent_server_healthy", [True, False])
+def test_launch_script_reports_reuse_only_for_a_healthy_agent_server(
+ tmp_path: Path, shell_environment: dict[str, str], agent_server_healthy: bool
+) -> None:
+ result = run_launch_script(tmp_path, shell_environment, agent_server_healthy=agent_server_healthy)
+
+ assert (AGENT_SERVER_PREFLIGHT_REUSE_MARKER in result.stdout.splitlines()) is agent_server_healthy
+
+
[email protected]("agent_server_healthy", [True, False])
+def test_launch_script_creates_an_agentsh_session_only_when_not_reusing(
+ tmp_path: Path, shell_environment: dict[str, str], agent_server_healthy: bool
+) -> None:
+ run_launch_script(tmp_path, shell_environment, agent_server_healthy=agent_server_healthy)
+
+ assert (tmp_path / SESSION_ID_FILE.lstrip("/")).exists() is not agent_server_healthy
+
+
@pytest.mark.parametrize(
"fault", ["install", "rename", "daemon_stop", "daemon_start", "health", "session", "empty_session", "null_session"]
)
@@ -298,7 +340,7 @@ def test_preparation_metric_measures_only_upload_and_execution(export_fails: boo
sandbox, "_agent_server_preflight", return_value=AgentServerPreflight(reused=True, capabilities=frozenset())
),
):
- clock.monotonic.side_effect = [10.0, 10.5, 11.25]
+ clock.monotonic.side_effect = itertools.chain([10.0, 10.0, 10.5, 11.25], itertools.repeat(20.0))
if export_fails:
metric_meter.side_effect = RuntimeError("export unavailable")
diff --git a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
index 70fea0e755ea..42709c0cf7f7 100644
--- a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
+++ b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
@@ -1,5 +1,7 @@
+import re
import json
import shlex
+import base64
import asyncio
import builtins
import subprocess
@@ -50,6 +52,7 @@ from products.tasks.backend.logic.services.agent_server_launcher import (
AGENT_SERVER_HEALTH_MAX_ATTEMPTS,
AGENT_SERVER_LAUNCH_CAPABILITIES,
AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX,
+ AGENT_SERVER_PREFLIGHT_CREDENTIAL_EXIT_CODE,
AGENT_SERVER_PREFLIGHT_REUSE_MARKER,
HOST_PRESSURE_PROBE_SCRIPT,
STARTUP_LOG_MAX_BYTES,
@@ -114,8 +117,13 @@ def _agent_server_launch_command(mock_execute: Any) -> str:
raise AssertionError("agent-server launch command not found among execute calls")
+def _decoded_command(command: str) -> str:
+ encoded = re.search(r"printf %s ([A-Za-z0-9+/=]+) \| base64 -d", command)
+ return base64.b64decode(encoded.group(1)).decode() if encoded else command
+
+
def _is_preflight_command(command: str) -> bool:
- return AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX in command
+ return AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX in _decoded_command(command)
def _preflight_stdout(*, reused: bool = False, capabilities: tuple[str, ...] = AGENT_SERVER_LAUNCH_CAPABILITIES) -> str:
@@ -661,7 +669,7 @@ class TestModalSandboxAgentServer:
mode="background",
)
- mock_setup.assert_called_once_with(None)
+ mock_setup.assert_not_called()
command = _agent_server_launch_command(mock_sandbox.execute)
import shlex
@@ -694,7 +702,7 @@ class TestModalSandboxAgentServer:
wait_for_health=False,
)
- commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]
+ commands = [_decoded_command(call.args[0]) for call in mock_sandbox.execute.call_args_list]
clear_index = next(
index for index, command in enumerate(commands) if "rm -rf" in command and "skills" in command
)
@@ -722,7 +730,7 @@ class TestModalSandboxAgentServer:
wait_for_health=False,
)
- commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]
+ commands = [_decoded_command(call.args[0]) for call in mock_sandbox.execute.call_args_list]
assert any("rm -rf" in command and "skills" in command for command in commands)
assert not any("./node_modules/.bin/agent-server" in command for command in commands)
@@ -761,7 +769,7 @@ class TestModalSandboxAgentServer:
allowed_domains=["example.com"],
)
- mock_setup_agentsh.assert_called_once_with(["example.com"])
+ mock_setup_agentsh.assert_not_called()
command = _agent_server_launch_command(mock_sandbox.execute)
assert "--createPr true" in command
assert "agentsh exec --client-timeout 2h --timeout 2h" in command
@@ -784,7 +792,7 @@ class TestModalSandboxAgentServer:
allowed_domains=[],
)
- mock_setup_agentsh.assert_called_once_with([])
+ mock_setup_agentsh.assert_not_called()
command = _agent_server_launch_command(mock_sandbox.execute)
assert "--allowedDomains" not in command
assert "agentsh exec --client-timeout 2h --timeout 2h" in command
@@ -1103,12 +1111,56 @@ class TestModalSandboxAgentServer:
commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]
# Bundled-skills clear, then refreshed bash-env and gh-guard files before reuse is accepted.
preflight_index = next(index for index, command in enumerate(commands) if _is_preflight_command(command))
- assert "task-filesystem.py enter" not in commands[preflight_index]
- assert "task-filesystem.py enter" in commands[preflight_index - 1]
- after_preflight = commands[preflight_index + 1 :]
- assert any("mv" in command and "/tmp/agentsh-bash-env.sh" in command for command in after_preflight)
- assert any("mv" in command and "/opt/posthog/bin/gh" in command for command in after_preflight)
- assert any(command == "chmod +x /opt/posthog/bin/gh" for command in after_preflight)
+ preflight = _decoded_command(commands[preflight_index])
+ probe_start = preflight.index(AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX)
+ assert preflight.index("task-filesystem.py enter -- /bin/bash -c") < probe_start
+ assert preflight.index("/tmp/agentsh-bash-env.sh") < probe_start
+ assert preflight.index("/opt/posthog/bin/gh") < probe_start
+
+ @pytest.mark.parametrize(
+ ("exit_code", "stderr", "expected_error"),
+ [
+ (
+ AGENT_SERVER_PREFLIGHT_CREDENTIAL_EXIT_CODE,
+ "__posthog_launch_preparation_failed=preflight\n",
+ ProcessTaskFatalError,
+ ),
+ (1, "__posthog_launch_preparation_failed=bundled_skills\n", RuntimeError),
+ (1, "__posthog_launch_preparation_failed=preflight\n", SandboxExecutionError),
+ ],
+ )
+ def test_start_agent_server_maps_combined_preparation_failures(
+ self, mock_sandbox: Any, exit_code: int, stderr: str, expected_error: type[Exception]
+ ) -> None:
+ mock_sandbox.execute = MagicMock(
+ return_value=ExecutionResult(stdout="", stderr=stderr, exit_code=exit_code, error=None)
+ )
+
+ with pytest.raises(expected_error) as raised:
+ mock_sandbox.start_agent_server(repository=None, task_id="task-123", run_id="run-456")
+
+ assert type(raised.value) is expected_error
+
+ @pytest.mark.parametrize(("domain_count", "uploads_script"), [(1, False), (400, True)])
+ def test_start_agent_server_uploads_launch_script_above_modal_command_limit(
+ self, mock_sandbox: Any, domain_count: int, uploads_script: bool
+ ) -> None:
+ mock_sandbox.execute = MagicMock(
+ return_value=ExecutionResult(stdout=_preflight_stdout(), stderr="", exit_code=0, error=None)
+ )
+ mock_sandbox.write_file = MagicMock(return_value=ExecutionResult(stdout="", stderr="", exit_code=0))
+ domains = [f"*.service-{index}-{'x' * 120}.example.com" for index in range(domain_count)]
+
+ mock_sandbox.start_agent_server(
+ repository=None, task_id="task-123", run_id="run-456", wait_for_health=False, allowed_domains=domains
+ )
+
+ preparation = next(
+ call.args[0]
+ for call in mock_sandbox.execute.call_args_list
+ if "posthog-launch-preparation-" in call.args[0]
+ )
+ assert preparation.startswith("bash /tmp/posthog-launch-preparation-") is uploads_script
def test_start_agent_server_relaunches_when_agentsh_is_unhealthy(self, mock_sandbox: Any):
def execute(command: str, timeout_seconds: int | None = None) -> ExecutionResult:
@@ -1196,7 +1248,7 @@ class TestModalSandboxAgentServer:
commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]
preflight = next(command for command in commands if _is_preflight_command(command))
- assert "xargs kill -KILL" in preflight
+ assert "xargs kill -KILL" in _decoded_command(preflight)
assert commands.index(preflight) < commands.index(_agent_server_launch_command(mock_sandbox.execute))
command = _agent_server_launch_command(mock_sandbox.execute)
assert health_ms == 250
@@ -1218,12 +1270,15 @@ class TestModalSandboxAgentServer:
commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]
launch_index = commands.index(_agent_server_launch_command(mock_sandbox.execute))
- assert launch_index == 3
- assert "task-filesystem.py enter" in commands[0]
- assert "curl" not in commands[0]
- assert _is_preflight_command(commands[1])
- assert "task-filesystem.py enter" not in commands[1]
- assert commands[2].startswith("bash /tmp/posthog-launch-preparation-")
+ assert launch_index == 1
+ assert _is_preflight_command(commands[0])
+ task_cleanup = next(
+ line
+ for line in _decoded_command(commands[0]).splitlines()
+ if "task-filesystem.py enter -- /bin/bash" in line
+ )
+ assert ENV_DISABLE_BUNDLED_SKILLS in task_cleanup
+ assert "curl" not in task_cleanup
def test_create_snapshot_waits_for_container_before_snapshot(self, mock_sandbox: Any) -> None:
events: list[str] = []
diff --git a/products/tasks/backend/tests/test_agentsh.py b/products/tasks/backend/tests/test_agentsh.py
index 1892b31ae4f4..24f091c96095 100644
--- a/products/tasks/backend/tests/test_agentsh.py
+++ b/products/tasks/backend/tests/test_agentsh.py
@@ -1,5 +1,7 @@
import os
+import re
import shlex
+import base64
import logging
import tempfile
import subprocess
@@ -636,6 +638,9 @@ class TestModalSandboxAgentShWrapping(TestCase):
)
def execute(command: str, timeout_seconds: int | None = None) -> ExecutionResult:
+ encoded_script = re.search(r"printf %s ([A-Za-z0-9+/=]+) \| base64 -d", command)
+ if encoded_script:
+ command = base64.b64decode(encoded_script.group(1)).decode()
if "--taskId" in command:
launched.append(command)
return ExecutionResult(stdout="", stderr="", exit_code=0)