:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(tasks): combine calls to modal (#120439)
Jonathan Mielood4c78756a516
diff --git a/products/tasks/backend/logic/services/agent_server_launcher.py b/products/tasks/backend/logic/services/agent_server_launcher.pyindex 8510573fbf64..ee0c41984374 100644--- a/products/tasks/backend/logic/services/agent_server_launcher.py+++ b/products/tasks/backend/logic/services/agent_server_launcher.py@@ -50,6 +50,7 @@ from products.tasks.backend.logic.services.sandbox import (CLAUDE_CREDENTIAL_UNAVAILABLE_MESSAGE,CODEX_CREDENTIAL_UNAVAILABLE_MESSAGE,WORKING_DIR,+ ExecutionResult,SandboxBase,build_agent_runtime_env_prefix,build_agent_server_capability_probe,@@ -213,6 +214,7 @@ class AgentServerPreflight:reused: boolcapabilities: frozenset[str]memory_watchdog_missing: bool = False+ launch_prepared: bool = Falsedef supports(self, capability: str) -> bool:return capability in self.capabilities@@ -221,6 +223,16 @@ class AgentServerPreflight:def build_agent_server_preflight_script(*, probe_health: bool, executable_paths: tuple[str, ...], probe_memory_watchdog: bool = False) -> str:+ lines = build_agent_server_preflight_lines(+ probe_health=probe_health, executable_paths=executable_paths, probe_memory_watchdog=probe_memory_watchdog+ )+ lines.append("exit 0")+ return "\n".join(lines)+++def build_agent_server_preflight_lines(+ *, probe_health: bool, executable_paths: tuple[str, ...], probe_memory_watchdog: bool = False+) -> list[str]:lines = [f"if ! chmod +x {shlex.quote(path)}; then exit {AGENT_SERVER_PREFLIGHT_CHMOD_EXIT_CODE}; fi"for path in executable_paths@@ -246,8 +258,7 @@ def build_agent_server_preflight_script(AGENT_SERVER_FREE_PORT_SCRIPT,])- lines.append("exit 0")- return "\n".join(lines)+ return linesCLAUDE_CREDENTIAL_UNAVAILABLE_MARKER = "claude_credential_unavailable"@@ -556,9 +567,6 @@ class AgentServerLaunchMixin(SandboxBase):def _agent_server_reuse_enabled(self) -> bool:return True- def _on_agent_server_reused(self) -> None:- return None-def _agent_server_preflight(self, allowed_domains: list[str] | None) -> AgentServerPreflight:executable_paths = self._install_agent_server_launch_files()result = self.execute_in_task_filesystem(@@ -574,6 +582,11 @@ class AgentServerLaunchMixin(SandboxBase):),timeout_seconds=AGENT_SERVER_PREFLIGHT_TIMEOUT_SECONDS,)+ return self._read_agent_server_preflight(result, executable_paths, allowed_domains)++ def _read_agent_server_preflight(+ self, result: ExecutionResult, executable_paths: tuple[str, ...], allowed_domains: list[str] | None+ ) -> AgentServerPreflight:if result.exit_code == AGENT_SERVER_PREFLIGHT_CHMOD_EXIT_CODE:raise SandboxExecutionError("Failed to set permissions on required sandbox file",@@ -603,7 +616,6 @@ class AgentServerLaunchMixin(SandboxBase):if AGENT_SERVER_PREFLIGHT_REUSE_MARKER in lines:if allowed_domains is None or self._agentsh_daemon_is_healthy():logger.info(f"Agent-server already healthy in sandbox {self.id}; skipping relaunch")- self._on_agent_server_reused()return AgentServerPreflight(reused=True, capabilities=capabilities)self._free_agent_server_port()return AgentServerPreflight(@@ -729,7 +741,8 @@ class AgentServerLaunchMixin(SandboxBase):org, repo = repository.lower().split("/")repo_path = f"/tmp/workspace/repos/{org}/{repo}"- self._prepare_agent_server_launch(allowed_domains)+ if not preflight.launch_prepared:+ self._prepare_agent_server_launch(allowed_domains)memory_watchdog_ready = self._install_memory_watchdog(preflight)codex_run_token_file = CODEX_RUN_TOKEN_FILE if codex_run_token else Nonediff --git a/products/tasks/backend/logic/services/modal_launch_preparation.py b/products/tasks/backend/logic/services/modal_launch_preparation.pyindex 51b1495b123f..fc5dbd73f598 100644--- a/products/tasks/backend/logic/services/modal_launch_preparation.py+++ b/products/tasks/backend/logic/services/modal_launch_preparation.py@@ -1,6 +1,8 @@import shleximport base64+from posthog.dataclasses import frozen+from products.tasks.backend.constants import DEFAULT_SANDBOX_WORKING_DIRfrom .agentsh import (@@ -15,25 +17,78 @@ from .agentsh import (generate_policy_yaml,read_gh_guard_script,)-from .task_filesystem import TASK_FILESYSTEM_SCRIPT, read_task_filesystem_script+from .task_filesystem import TASK_FILESYSTEM_SCRIPT, read_task_filesystem_script, task_filesystem_command+++@frozen+class LaunchFile:+ path: str+ payload: bytes+ mode: strdef build_modal_launch_preparation_script(allowed_domains: list[str] | None) -> str:- files = [- (BASH_ENV_SCRIPT, generate_bash_env_script().encode(), "644"),- (GH_GUARD_INSTALL_PATH, read_gh_guard_script(), "755"),- (TASK_FILESYSTEM_SCRIPT, read_task_filesystem_script(), "644"),- ]+ files = _base_files()if allowed_domains is not None:- files.extend(- [- ("/etc/agentsh/config.yaml", generate_config_yaml(enable_ptrace=True, full_trace=True).encode(), "644"),- ("/etc/agentsh/policies/default.yaml", generate_policy_yaml(allowed_domains).encode(), "644"),- (ENV_WRAPPER_SCRIPT, generate_env_wrapper().encode(), "755"),- ]- )+ files.extend(_agentsh_files(allowed_domains))+ script = _script_header()+ script.extend(_install_lines(files))+ script.extend(_installed_lines())+ if allowed_domains is not None:+ script.extend(_daemon_session_lines())+ return "\n".join(script) + "\n"+++def build_modal_agent_server_launch_script(+ allowed_domains: list[str] | None, *, bundled_skills_clear_command: str, preflight_lines: list[str]+) -> str:+ script = _script_header()+ script.extend(_install_lines(_base_files()))+ script.extend(_installed_lines())+ script.extend(+ [+ "stage=bundled_skills",+ f"(umask 022 && {task_filesystem_command(bundled_skills_clear_command)})",+ "stage=preflight",+ "set +euo pipefail",+ *preflight_lines,+ "set -euo pipefail",+ ]+ )+ if allowed_domains is not None:+ script.append("stage=install")+ script.extend(_install_lines(_agentsh_files(allowed_domains)))+ script.append("installed_at=$(date +%s%3N)")+ script.extend(_daemon_session_lines())+ return "\n".join(script) + "\n"+++def _base_files() -> list[LaunchFile]:+ return [+ LaunchFile(path=BASH_ENV_SCRIPT, payload=generate_bash_env_script().encode(), mode="644"),+ LaunchFile(path=GH_GUARD_INSTALL_PATH, payload=read_gh_guard_script(), mode="755"),+ LaunchFile(path=TASK_FILESYSTEM_SCRIPT, payload=read_task_filesystem_script(), mode="644"),+ ]+- script = [+def _agentsh_files(allowed_domains: list[str]) -> list[LaunchFile]:+ return [+ LaunchFile(+ path="/etc/agentsh/config.yaml",+ payload=generate_config_yaml(enable_ptrace=True, full_trace=True).encode(),+ mode="644",+ ),+ LaunchFile(+ path="/etc/agentsh/policies/default.yaml",+ payload=generate_policy_yaml(allowed_domains).encode(),+ mode="644",+ ),+ LaunchFile(path=ENV_WRAPPER_SCRIPT, payload=generate_env_wrapper().encode(), mode="755"),+ ]+++def _script_header() -> list[str]:+ return ["#!/bin/bash","set -euo pipefail","export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",@@ -60,59 +115,63 @@ def build_modal_launch_preparation_script(allowed_domains: list[str] | None) ->"}","trap cleanup EXIT",]- for path, payload, mode in files:- destination = shlex.quote(path)- encoded = shlex.quote(base64.b64encode(payload).decode())- script.extend(+++def _install_lines(files: list[LaunchFile]) -> list[str]:+ lines: list[str] = ["first_staged=${#staged_files[@]}"]+ for file in files:+ destination = shlex.quote(file.path)+ encoded = shlex.quote(base64.b64encode(file.payload).decode())+ lines.extend([f'mkdir -p -- "$(dirname -- {destination})"',- f"temporary=$(mktemp {shlex.quote(path + '.tmp.XXXXXX')})",+ f"temporary=$(mktemp {shlex.quote(file.path + '.tmp.XXXXXX')})",'staged_files+=("$temporary")',f'printf %s {encoded} | base64 -d > "$temporary"',- f'chmod {mode} "$temporary"',+ f'chmod {file.mode} "$temporary"',])- for index, (path, _, _) in enumerate(files):- script.append(f"test ! -d {shlex.quote(path)}")- script.append(f'mv -f -- "${{staged_files[{index}]}}" {shlex.quote(path)}')- script.extend(- [- "installed_at=$(date +%s%3N)",- 'printf "__posthog_launch_preparation_install_ms=%s\\n" "$((installed_at - started_at))"',- ]- )+ for index, file in enumerate(files):+ lines.append(f"test ! -d {shlex.quote(file.path)}")+ lines.append(f'mv -f -- "${{staged_files[$((first_staged + {index}))]}}" {shlex.quote(file.path)}')+ return lines- if allowed_domains is not None:- script.extend(- [- "stage=daemon_session",- "pkill -f '[a]gentsh server' || [ $? -eq 1 ]",- "for i in $(seq 1 50); do",- " if ! pgrep -f '[a]gentsh server' > /dev/null; then break; fi",- " sleep 0.1",- "done",- "if pgrep -f '[a]gentsh server' > /dev/null; then exit 1; fi",- "mkdir -p /var/log/agentsh /var/lib/agentsh/sessions",- f"rm -f -- {shlex.quote(SESSION_ID_FILE)}",- f"nohup /usr/bin/python3 -I {TASK_FILESYSTEM_SCRIPT} enter -- /usr/bin/agentsh server --config /etc/agentsh/config.yaml > /var/log/agentsh/agentsh.log 2>&1 &",- "daemon_pid=$!",- "healthy=0",- "for i in $(seq 1 30); do",- f" if [ \"$(curl -s --max-time 1 -o /dev/null -w '%{{http_code}}' http://127.0.0.1:{AGENTSH_DAEMON_PORT}/health || true)\" = 200 ]; then",- " healthy=1",- " break",- " fi",- " sleep 0.5",- "done",- '[ "$healthy" -eq 1 ]',- 'kill -0 "$daemon_pid"',- f"session_id=$(timeout 60s /usr/bin/agentsh session create --workspace {shlex.quote(DEFAULT_SANDBOX_WORKING_DIR)} --policy default --json | jq -er '.id | select(type == \"string\" and length > 0)')",- f"temporary=$(mktemp {shlex.quote(SESSION_ID_FILE + '.tmp.XXXXXX')})",- 'staged_files+=("$temporary")',- 'printf %s "$session_id" > "$temporary"',- f'mv -f -- "$temporary" {shlex.quote(SESSION_ID_FILE)}',- f"test -s {shlex.quote(SESSION_ID_FILE)}",- 'printf "__posthog_launch_preparation_daemon_session_ms=%s\\n" "$(($(date +%s%3N) - installed_at))"',- ]- )- return "\n".join(script) + "\n"++def _installed_lines() -> list[str]:+ return [+ "installed_at=$(date +%s%3N)",+ 'printf "__posthog_launch_preparation_install_ms=%s\\n" "$((installed_at - started_at))"',+ ]+++def _daemon_session_lines() -> list[str]:+ return [+ "stage=daemon_session",+ "pkill -f '[a]gentsh server' || [ $? -eq 1 ]",+ "for i in $(seq 1 50); do",+ " if ! pgrep -f '[a]gentsh server' > /dev/null; then break; fi",+ " sleep 0.1",+ "done",+ "if pgrep -f '[a]gentsh server' > /dev/null; then exit 1; fi",+ "mkdir -p /var/log/agentsh /var/lib/agentsh/sessions",+ f"rm -f -- {shlex.quote(SESSION_ID_FILE)}",+ f"nohup /usr/bin/python3 -I {TASK_FILESYSTEM_SCRIPT} enter -- /usr/bin/agentsh server --config /etc/agentsh/config.yaml > /var/log/agentsh/agentsh.log 2>&1 &",+ "daemon_pid=$!",+ "healthy=0",+ "for i in $(seq 1 30); do",+ f" if [ \"$(curl -s --max-time 1 -o /dev/null -w '%{{http_code}}' http://127.0.0.1:{AGENTSH_DAEMON_PORT}/health || true)\" = 200 ]; then",+ " healthy=1",+ " break",+ " fi",+ " sleep 0.5",+ "done",+ '[ "$healthy" -eq 1 ]',+ 'kill -0 "$daemon_pid"',+ f"session_id=$(timeout 60s /usr/bin/agentsh session create --workspace {shlex.quote(DEFAULT_SANDBOX_WORKING_DIR)} --policy default --json | jq -er '.id | select(type == \"string\" and length > 0)')",+ f"temporary=$(mktemp {shlex.quote(SESSION_ID_FILE + '.tmp.XXXXXX')})",+ 'staged_files+=("$temporary")',+ 'printf %s "$session_id" > "$temporary"',+ f'mv -f -- "$temporary" {shlex.quote(SESSION_ID_FILE)}',+ f"test -s {shlex.quote(SESSION_ID_FILE)}",+ 'printf "__posthog_launch_preparation_daemon_session_ms=%s\\n" "$(($(date +%s%3N) - installed_at))"',+ ]diff --git a/products/tasks/backend/logic/services/modal_sandbox.py b/products/tasks/backend/logic/services/modal_sandbox.pyindex a2d86a238fb2..f8a619baaf61 100644--- a/products/tasks/backend/logic/services/modal_sandbox.py+++ b/products/tasks/backend/logic/services/modal_sandbox.py@@ -11,7 +11,8 @@ import asyncioimport loggingimport tempfileimport threading-from collections.abc import Iterable+import dataclasses+from collections.abc import Callable, Iterablefrom dataclasses import dataclassfrom functools import lru_cachefrom http import HTTPStatus@@ -60,12 +61,6 @@ from products.tasks.backend.exceptions import (SnapshotFileLimitExceededError,SnapshotTimeoutError,)-from products.tasks.backend.logic.services.agentsh import (- BASH_ENV_SCRIPT,- GH_GUARD_INSTALL_PATH,- generate_bash_env_script,- read_gh_guard_script,-)from products.tasks.backend.logic.services.cpu_billing import (CPU_BILLING_SAMPLER_PATH as CPU_BILLING_SAMPLER_PATH,CPU_BILLING_STATE_PATH,@@ -84,13 +79,16 @@ from products.tasks.backend.logic.services.local_skills import (LocalSkillsCache,populate_skills_directory,)-from products.tasks.backend.logic.services.modal_launch_preparation import build_modal_launch_preparation_script+from products.tasks.backend.logic.services.modal_launch_preparation import (+ build_modal_agent_server_launch_script,+ build_modal_launch_preparation_script,+)from products.tasks.backend.logic.services.modal_provision_diagnostics import (SandboxProvisionDiagnostics,capture_modal_output_if_debug,summarize_modal_output,)-from products.tasks.backend.logic.services.sandbox import redact_sandbox_command+from products.tasks.backend.logic.services.sandbox import build_bundled_skills_clear_command, redact_sandbox_commandfrom products.tasks.backend.logic.services.task_filesystem import (TASK_FILESYSTEM_SCRIPT,TASK_FILESYSTEM_VERSION,@@ -101,8 +99,13 @@ from products.tasks.backend.models import DevStackImageTarget, SandboxSnapshotfrom .agent_server_launcher import (AGENT_SERVER_PORT as AGENT_SERVER_PORT,+ AGENT_SERVER_PREFLIGHT_CREDENTIAL_EXIT_CODE,+ AGENT_SERVER_PREFLIGHT_REUSE_MARKER,+ AGENT_SERVER_PREFLIGHT_TIMEOUT_SECONDS,AgentServerLaunchMixin,+ AgentServerPreflight,_session_init_probe_hosts as _session_init_probe_hosts,+ build_agent_server_preflight_lines,)from .sandbox import (AgentServerResult,@@ -185,6 +188,17 @@ _EMBEDDED_PROXY_STATUS = re.compile(r"\b([45]\d{2})\b")ControlPlaneFailure = Literal["rate_limited", "unavailable"]RUNNING_STATUS_CACHE_SECONDS = 10.0+MODAL_LAUNCH_PREPARATION_TIMEOUT_SECONDS = 120+MODAL_INLINE_COMMAND_MAX_BYTES = 60_000+++def _launch_preparation_failed_stage(stderr: str) -> str:+ stage = re.search(+ r"^__posthog_launch_preparation_failed=(install|bundled_skills|preflight|daemon_session)$",+ stderr,+ re.MULTILINE,+ )+ return stage.group(1) if stage else "unknown"def _classify_control_plane_failure(error: BaseException) -> ControlPlaneFailure | None:@@ -829,8 +843,9 @@ class ModalSandbox(AgentServerLaunchMixin):self.provision_diagnostics = Noneself._destroyed = False- def _install_agent_server_launch_files(self) -> tuple[str, ...]:- return ()+ def _validate_agent_server_launch(self) -> None:+ if not self._is_running_cached():+ raise RuntimeError("Sandbox not in running state.")def _wrap_agent_server_command(self, command: str) -> str:return f"/usr/bin/python3 -I {TASK_FILESYSTEM_SCRIPT} enter -- {command}"@@ -865,28 +880,72 @@ class ModalSandbox(AgentServerLaunchMixin):def _sandbox_runtime(self) -> str | None:return "vm" if self.config.is_vm else "gvisor"- def _on_agent_server_reused(self) -> None:- # A restored snapshot can carry a healthy agent-server with a stale bash-env- # or gh shim from the snapshot's epoch. Refresh both before accepting reuse;- # agentsh setup and session replacement stay on the fresh-launch path so- # reuse doesn't disrupt the running server or its agentsh session.- self._write_required_file(BASH_ENV_SCRIPT, generate_bash_env_script().encode())- self._write_required_file(GH_GUARD_INSTALL_PATH, read_gh_guard_script())- self._chmod_required(GH_GUARD_INSTALL_PATH, "+x")+ def _agent_server_preflight(self, allowed_domains: list[str] | None) -> AgentServerPreflight:+ script = build_modal_agent_server_launch_script(+ allowed_domains,+ bundled_skills_clear_command=build_bundled_skills_clear_command(),+ preflight_lines=build_agent_server_preflight_lines(+ probe_health=self._agent_server_reuse_enabled(),+ executable_paths=(),+ probe_memory_watchdog=settings.TASKS_SANDBOX_MEMORY_WATCHDOG_ENABLED,+ ),+ )+ timeout_seconds = MODAL_LAUNCH_PREPARATION_TIMEOUT_SECONDS + AGENT_SERVER_PREFLIGHT_TIMEOUT_SECONDS+ encoded_script = base64.b64encode(script.encode()).decode()+ command = (+ "script_path=$(mktemp /tmp/posthog-launch-preparation-XXXXXXXX) && "+ f'printf %s {encoded_script} | base64 -d > "$script_path" && bash "$script_path"'+ )++ def run() -> ExecutionResult:+ if len(command.encode()) <= MODAL_INLINE_COMMAND_MAX_BYTES:+ return self.execute(command, timeout_seconds=timeout_seconds)+ script_path = f"/tmp/posthog-launch-preparation-{uuid.uuid4().hex}.sh"+ self._write_required_file(script_path, script.encode())+ return self.execute(f"bash {shlex.quote(script_path)}", timeout_seconds=timeout_seconds)++ result, total_ms = self._run_launch_preparation(+ run, completed_exit_codes=(0, AGENT_SERVER_PREFLIGHT_CREDENTIAL_EXIT_CODE)+ )+ if _launch_preparation_failed_stage(result.stderr) == "bundled_skills":+ raise RuntimeError(f"Failed to clear bundled skills in sandbox {self.id}: {result.stderr}")+ if result.exit_code != AGENT_SERVER_PREFLIGHT_CREDENTIAL_EXIT_CODE:+ self._finish_launch_preparation(result, upload_ms=0, total_ms=total_ms)+ preflight = self._read_agent_server_preflight(result, (), allowed_domains)+ if AGENT_SERVER_PREFLIGHT_REUSE_MARKER in result.stdout.splitlines():+ return preflight+ return dataclasses.replace(preflight, launch_prepared=True)def _prepare_agent_server_launch(self, allowed_domains: list[str] | None) -> None:script_path = f"/tmp/posthog-launch-preparation-{uuid.uuid4().hex}.sh"+ upload_ms = 0++ def upload_and_run() -> ExecutionResult:+ nonlocal upload_ms+ started_at = time.monotonic()+ self._write_required_file(script_path, build_modal_launch_preparation_script(allowed_domains).encode())+ upload_ms = round((time.monotonic() - started_at) * 1000)+ return self.execute(+ f"bash {shlex.quote(script_path)}", timeout_seconds=MODAL_LAUNCH_PREPARATION_TIMEOUT_SECONDS+ )++ result, total_ms = self._run_launch_preparation(upload_and_run, completed_exit_codes=(0,))+ self._finish_launch_preparation(result, upload_ms=upload_ms, total_ms=total_ms)++ def _run_launch_preparation(+ self, run: Callable[[], ExecutionResult], *, completed_exit_codes: tuple[int, ...]+ ) -> tuple[ExecutionResult, int]:started_at = time.monotonic()result: ExecutionResult | None = Nonetry:- self._write_required_file(script_path, build_modal_launch_preparation_script(allowed_domains).encode())- uploaded_at = time.monotonic()- result = self.execute(f"bash {shlex.quote(script_path)}", timeout_seconds=120)+ result = run()finally:total_ms = round((time.monotonic() - started_at) * 1000)- record_launch_preparation_ms(- total_ms, "COMPLETED" if result is not None and result.exit_code == 0 else "FAILED"- )+ completed = result is not None and result.exit_code in completed_exit_codes+ record_launch_preparation_ms(total_ms, "COMPLETED" if completed else "FAILED")+ return result, total_ms++ def _finish_launch_preparation(self, result: ExecutionResult, *, upload_ms: int, total_ms: int) -> None:timings = {stage: int(duration)for stage, duration in re.findall(@@ -897,17 +956,14 @@ class ModalSandbox(AgentServerLaunchMixin):"Modal launch preparation finished in sandbox %s: upload_ms=%d install_ms=%s ""daemon_session_ms=%s total_ms=%d exit_code=%d",self.id,- round((uploaded_at - started_at) * 1000),+ upload_ms,timings.get("install_ms"),timings.get("daemon_session_ms"),total_ms,result.exit_code,)if result.exit_code != 0:- stage = re.search(- r"^__posthog_launch_preparation_failed=(install|daemon_session)$", result.stderr, re.MULTILINE- )- preparation_stage = stage.group(1) if stage else "unknown"+ preparation_stage = _launch_preparation_failed_stage(result.stderr)agentsh_log = ""if preparation_stage == "daemon_session":try:diff --git a/products/tasks/backend/logic/services/tests/test_modal_launch_preparation.py b/products/tasks/backend/logic/services/tests/test_modal_launch_preparation.pyindex b9796c5f5c5b..f1117921de13 100644--- a/products/tasks/backend/logic/services/tests/test_modal_launch_preparation.py+++ b/products/tasks/backend/logic/services/tests/test_modal_launch_preparation.py@@ -2,6 +2,7 @@ import osimport reimport shleximport signal+import itertoolsimport subprocessfrom datetime import timedeltafrom pathlib import Path@@ -10,7 +11,11 @@ import pytestfrom unittest.mock import MagicMock, patchfrom products.tasks.backend.exceptions import SandboxExecutionError-from products.tasks.backend.logic.services.agent_server_launcher import AgentServerPreflight+from products.tasks.backend.logic.services.agent_server_launcher import (+ AGENT_SERVER_PREFLIGHT_REUSE_MARKER,+ AgentServerPreflight,+ build_agent_server_preflight_lines,+)from products.tasks.backend.logic.services.agentsh import (BASH_ENV_SCRIPT,ENV_WRAPPER_SCRIPT,@@ -26,13 +31,17 @@ from products.tasks.backend.logic.services.launch_preparation_metrics import (launch_preparation_metric_context,record_launch_preparation_ms,)-from products.tasks.backend.logic.services.modal_launch_preparation import build_modal_launch_preparation_script+from products.tasks.backend.logic.services.modal_launch_preparation import (+ build_modal_agent_server_launch_script,+ build_modal_launch_preparation_script,+)from products.tasks.backend.logic.services.modal_sandbox import ModalSandboxfrom products.tasks.backend.logic.services.sandbox import (ExecutionResult,SandboxConfig,build_bundled_skills_clear_command,)+from products.tasks.backend.logic.services.task_filesystem import TASK_FILESYSTEM_SCRIPT@pytest.fixture@@ -69,9 +78,9 @@ esacdef run_preparation(- root: Path, allowed_domains: list[str] | None, environment: dict[str, str]+ root: Path, allowed_domains: list[str] | None, environment: dict[str, str], script: str | None = None) -> subprocess.CompletedProcess[str]:- script = build_modal_launch_preparation_script(allowed_domains)+ script = script if script is not None else build_modal_launch_preparation_script(allowed_domains)script = re.sub(r"/(?:tmp|etc|opt|var)/", lambda match: f"{root}{match.group()}", script)script = script.replace("export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",@@ -132,6 +141,39 @@ def test_preparation_installs_complete_files_and_can_be_reapplied(assert not (tmp_path / SESSION_ID_FILE.lstrip("/")).exists()+def run_launch_script(+ root: Path, environment: dict[str, str], *, agent_server_healthy: bool+) -> subprocess.CompletedProcess[str]:+ health_body = '{"status":"ok","hasSession":true}' if agent_server_healthy else '{"status":"ok","hasSession":false}'+ (root / "bin/curl").write_text(+ f'#!/bin/bash\ncase "$*" in *localhost:8080/health*) echo {shlex.quote(health_body)} ;; *) echo 200 ;; esac\n'+ )+ script = build_modal_agent_server_launch_script(+ ["example.com"],+ bundled_skills_clear_command="true",+ preflight_lines=build_agent_server_preflight_lines(probe_health=True, executable_paths=()),+ ).replace(f"/usr/bin/python3 -I {TASK_FILESYSTEM_SCRIPT} enter -- ", "")+ return run_preparation(root, ["example.com"], environment, script)++[email protected]("agent_server_healthy", [True, False])+def test_launch_script_reports_reuse_only_for_a_healthy_agent_server(+ tmp_path: Path, shell_environment: dict[str, str], agent_server_healthy: bool+) -> None:+ result = run_launch_script(tmp_path, shell_environment, agent_server_healthy=agent_server_healthy)++ assert (AGENT_SERVER_PREFLIGHT_REUSE_MARKER in result.stdout.splitlines()) is agent_server_healthy++[email protected]("agent_server_healthy", [True, False])+def test_launch_script_creates_an_agentsh_session_only_when_not_reusing(+ tmp_path: Path, shell_environment: dict[str, str], agent_server_healthy: bool+) -> None:+ run_launch_script(tmp_path, shell_environment, agent_server_healthy=agent_server_healthy)++ assert (tmp_path / SESSION_ID_FILE.lstrip("/")).exists() is not agent_server_healthy++@pytest.mark.parametrize("fault", ["install", "rename", "daemon_stop", "daemon_start", "health", "session", "empty_session", "null_session"])@@ -298,7 +340,7 @@ def test_preparation_metric_measures_only_upload_and_execution(export_fails: boosandbox, "_agent_server_preflight", return_value=AgentServerPreflight(reused=True, capabilities=frozenset())),):- clock.monotonic.side_effect = [10.0, 10.5, 11.25]+ clock.monotonic.side_effect = itertools.chain([10.0, 10.0, 10.5, 11.25], itertools.repeat(20.0))if export_fails:metric_meter.side_effect = RuntimeError("export unavailable")diff --git a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py b/products/tasks/backend/logic/services/tests/test_modal_sandbox.pyindex 70fea0e755ea..42709c0cf7f7 100644--- a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py+++ b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py@@ -1,5 +1,7 @@+import reimport jsonimport shlex+import base64import asyncioimport builtinsimport subprocess@@ -50,6 +52,7 @@ from products.tasks.backend.logic.services.agent_server_launcher import (AGENT_SERVER_HEALTH_MAX_ATTEMPTS,AGENT_SERVER_LAUNCH_CAPABILITIES,AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX,+ AGENT_SERVER_PREFLIGHT_CREDENTIAL_EXIT_CODE,AGENT_SERVER_PREFLIGHT_REUSE_MARKER,HOST_PRESSURE_PROBE_SCRIPT,STARTUP_LOG_MAX_BYTES,@@ -114,8 +117,13 @@ def _agent_server_launch_command(mock_execute: Any) -> str:raise AssertionError("agent-server launch command not found among execute calls")+def _decoded_command(command: str) -> str:+ encoded = re.search(r"printf %s ([A-Za-z0-9+/=]+) \| base64 -d", command)+ return base64.b64decode(encoded.group(1)).decode() if encoded else command++def _is_preflight_command(command: str) -> bool:- return AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX in command+ return AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX in _decoded_command(command)def _preflight_stdout(*, reused: bool = False, capabilities: tuple[str, ...] = AGENT_SERVER_LAUNCH_CAPABILITIES) -> str:@@ -661,7 +669,7 @@ class TestModalSandboxAgentServer:mode="background",)- mock_setup.assert_called_once_with(None)+ mock_setup.assert_not_called()command = _agent_server_launch_command(mock_sandbox.execute)import shlex@@ -694,7 +702,7 @@ class TestModalSandboxAgentServer:wait_for_health=False,)- commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]+ commands = [_decoded_command(call.args[0]) for call in mock_sandbox.execute.call_args_list]clear_index = next(index for index, command in enumerate(commands) if "rm -rf" in command and "skills" in command)@@ -722,7 +730,7 @@ class TestModalSandboxAgentServer:wait_for_health=False,)- commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]+ commands = [_decoded_command(call.args[0]) for call in mock_sandbox.execute.call_args_list]assert any("rm -rf" in command and "skills" in command for command in commands)assert not any("./node_modules/.bin/agent-server" in command for command in commands)@@ -761,7 +769,7 @@ class TestModalSandboxAgentServer:allowed_domains=["example.com"],)- mock_setup_agentsh.assert_called_once_with(["example.com"])+ mock_setup_agentsh.assert_not_called()command = _agent_server_launch_command(mock_sandbox.execute)assert "--createPr true" in commandassert "agentsh exec --client-timeout 2h --timeout 2h" in command@@ -784,7 +792,7 @@ class TestModalSandboxAgentServer:allowed_domains=[],)- mock_setup_agentsh.assert_called_once_with([])+ mock_setup_agentsh.assert_not_called()command = _agent_server_launch_command(mock_sandbox.execute)assert "--allowedDomains" not in commandassert "agentsh exec --client-timeout 2h --timeout 2h" in command@@ -1103,12 +1111,56 @@ class TestModalSandboxAgentServer:commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]# Bundled-skills clear, then refreshed bash-env and gh-guard files before reuse is accepted.preflight_index = next(index for index, command in enumerate(commands) if _is_preflight_command(command))- assert "task-filesystem.py enter" not in commands[preflight_index]- assert "task-filesystem.py enter" in commands[preflight_index - 1]- after_preflight = commands[preflight_index + 1 :]- assert any("mv" in command and "/tmp/agentsh-bash-env.sh" in command for command in after_preflight)- assert any("mv" in command and "/opt/posthog/bin/gh" in command for command in after_preflight)- assert any(command == "chmod +x /opt/posthog/bin/gh" for command in after_preflight)+ preflight = _decoded_command(commands[preflight_index])+ probe_start = preflight.index(AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX)+ assert preflight.index("task-filesystem.py enter -- /bin/bash -c") < probe_start+ assert preflight.index("/tmp/agentsh-bash-env.sh") < probe_start+ assert preflight.index("/opt/posthog/bin/gh") < probe_start++ @pytest.mark.parametrize(+ ("exit_code", "stderr", "expected_error"),+ [+ (+ AGENT_SERVER_PREFLIGHT_CREDENTIAL_EXIT_CODE,+ "__posthog_launch_preparation_failed=preflight\n",+ ProcessTaskFatalError,+ ),+ (1, "__posthog_launch_preparation_failed=bundled_skills\n", RuntimeError),+ (1, "__posthog_launch_preparation_failed=preflight\n", SandboxExecutionError),+ ],+ )+ def test_start_agent_server_maps_combined_preparation_failures(+ self, mock_sandbox: Any, exit_code: int, stderr: str, expected_error: type[Exception]+ ) -> None:+ mock_sandbox.execute = MagicMock(+ return_value=ExecutionResult(stdout="", stderr=stderr, exit_code=exit_code, error=None)+ )++ with pytest.raises(expected_error) as raised:+ mock_sandbox.start_agent_server(repository=None, task_id="task-123", run_id="run-456")++ assert type(raised.value) is expected_error++ @pytest.mark.parametrize(("domain_count", "uploads_script"), [(1, False), (400, True)])+ def test_start_agent_server_uploads_launch_script_above_modal_command_limit(+ self, mock_sandbox: Any, domain_count: int, uploads_script: bool+ ) -> None:+ mock_sandbox.execute = MagicMock(+ return_value=ExecutionResult(stdout=_preflight_stdout(), stderr="", exit_code=0, error=None)+ )+ mock_sandbox.write_file = MagicMock(return_value=ExecutionResult(stdout="", stderr="", exit_code=0))+ domains = [f"*.service-{index}-{'x' * 120}.example.com" for index in range(domain_count)]++ mock_sandbox.start_agent_server(+ repository=None, task_id="task-123", run_id="run-456", wait_for_health=False, allowed_domains=domains+ )++ preparation = next(+ call.args[0]+ for call in mock_sandbox.execute.call_args_list+ if "posthog-launch-preparation-" in call.args[0]+ )+ assert preparation.startswith("bash /tmp/posthog-launch-preparation-") is uploads_scriptdef test_start_agent_server_relaunches_when_agentsh_is_unhealthy(self, mock_sandbox: Any):def execute(command: str, timeout_seconds: int | None = None) -> ExecutionResult:@@ -1196,7 +1248,7 @@ class TestModalSandboxAgentServer:commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]preflight = next(command for command in commands if _is_preflight_command(command))- assert "xargs kill -KILL" in preflight+ assert "xargs kill -KILL" in _decoded_command(preflight)assert commands.index(preflight) < commands.index(_agent_server_launch_command(mock_sandbox.execute))command = _agent_server_launch_command(mock_sandbox.execute)assert health_ms == 250@@ -1218,12 +1270,15 @@ class TestModalSandboxAgentServer:commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]launch_index = commands.index(_agent_server_launch_command(mock_sandbox.execute))- assert launch_index == 3- assert "task-filesystem.py enter" in commands[0]- assert "curl" not in commands[0]- assert _is_preflight_command(commands[1])- assert "task-filesystem.py enter" not in commands[1]- assert commands[2].startswith("bash /tmp/posthog-launch-preparation-")+ assert launch_index == 1+ assert _is_preflight_command(commands[0])+ task_cleanup = next(+ line+ for line in _decoded_command(commands[0]).splitlines()+ if "task-filesystem.py enter -- /bin/bash" in line+ )+ assert ENV_DISABLE_BUNDLED_SKILLS in task_cleanup+ assert "curl" not in task_cleanupdef test_create_snapshot_waits_for_container_before_snapshot(self, mock_sandbox: Any) -> None:events: list[str] = []diff --git a/products/tasks/backend/tests/test_agentsh.py b/products/tasks/backend/tests/test_agentsh.pyindex 1892b31ae4f4..24f091c96095 100644--- a/products/tasks/backend/tests/test_agentsh.py+++ b/products/tasks/backend/tests/test_agentsh.py@@ -1,5 +1,7 @@import os+import reimport shlex+import base64import loggingimport tempfileimport subprocess@@ -636,6 +638,9 @@ class TestModalSandboxAgentShWrapping(TestCase):)def execute(command: str, timeout_seconds: int | None = None) -> ExecutionResult:+ encoded_script = re.search(r"printf %s ([A-Za-z0-9+/=]+) \| base64 -d", command)+ if encoded_script:+ command = base64.b64decode(encoded_script.group(1)).decode()if "--taskId" in command:launched.append(command)return ExecutionResult(stdout="", stderr="", exit_code=0)