Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): retry research after a transient sandbox failure (#123983)

Alessandro Pogliaghie5c8eb532c0c
diff --git a/products/signals/backend/management/commands/rerun_signal_report.py b/products/signals/backend/management/commands/rerun_signal_report.py
index 029ba4616a83..e0e0a23b988b 100644
--- a/products/signals/backend/management/commands/rerun_signal_report.py
+++ b/products/signals/backend/management/commands/rerun_signal_report.py
@@ -25,7 +25,7 @@ from products.signals.backend.models import SignalReport
from products.signals.backend.quota import self_driving_quota_gate
from products.signals.backend.supersession import pending_replacement
from products.signals.backend.task_run_artefacts import SIGNALS_PRODUCT, TASK_RUN_TYPE_IMPLEMENTATION
-from products.signals.backend.temporal.summary import SignalReportSummaryWorkflow
+from products.signals.backend.temporal.summary import RESEARCH_SANDBOX_RETRY_BUDGET, SignalReportSummaryWorkflow
from products.signals.backend.temporal.types import IMPLEMENTATION_DEBOUNCE_SECONDS, SignalReportSummaryWorkflowInputs
from products.tasks.backend.facade import api as tasks_facade
from products.tasks.backend.facade.usage import task_run_usage_limited
@@ -74,7 +74,8 @@ class Command(BaseCommand):
id=SignalReportSummaryWorkflow.workflow_id_for(inputs.team_id, inputs.report_id),
task_queue=settings.VIDEO_EXPORT_TASK_QUEUE,
id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE,
- execution_timeout=timedelta(hours=1, seconds=IMPLEMENTATION_DEBOUNCE_SECONDS),
+ execution_timeout=timedelta(hours=1, seconds=IMPLEMENTATION_DEBOUNCE_SECONDS)
+ + RESEARCH_SANDBOX_RETRY_BUDGET,
)
def handle(self, *args: object, **options: object) -> None:
diff --git a/products/signals/backend/temporal/agentic/report.py b/products/signals/backend/temporal/agentic/report.py
index c4fbd5560cf6..03f191dabfe0 100644
--- a/products/signals/backend/temporal/agentic/report.py
+++ b/products/signals/backend/temporal/agentic/report.py
@@ -84,7 +84,7 @@ from products.signals.backend.typed_report_links import (
outgoing_links,
)
from products.tasks.backend.facade import api as tasks_facade
-from products.tasks.backend.facade.agents import CustomPromptSandboxContext
+from products.tasks.backend.facade.agents import CustomPromptSandboxContext, SandboxRunFailed
logger = structlog.get_logger(__name__)
@@ -1097,6 +1097,9 @@ def _capture_research_steering_attached(*, team_id: int, report_id: str, steerin
logger.exception("Failed to capture signals_research_steering_attached", report_id=report_id)
+RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE = "ResearchSandboxUnavailable"
+
+
@temporalio.activity.defn
@scoped_temporal()
@close_db_connections
@@ -1271,6 +1274,19 @@ async def run_agentic_report_activity(input: RunAgenticReportInput) -> RunAgenti
research_task_id=result.research_task_id,
charts_enabled=True,
)
+ except SandboxRunFailed as error:
+ logger.exception(
+ "signals agentic report failed",
+ report_id=input.report_id,
+ team_id=input.team_id,
+ error=str(error),
+ sandbox_error_type=error.error_type,
+ )
+ if error.transient:
+ raise temporalio.exceptions.ApplicationError(
+ str(error), type=RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE
+ ) from error
+ raise
except Exception as error:
logger.exception(
"signals agentic report failed",
diff --git a/products/signals/backend/temporal/grouping.py b/products/signals/backend/temporal/grouping.py
index 61b5c53af9e4..18f3f81f7130 100644
--- a/products/signals/backend/temporal/grouping.py
+++ b/products/signals/backend/temporal/grouping.py
@@ -66,7 +66,7 @@ from products.signals.backend.temporal.signal_queries import (
soft_delete_report_signals,
wait_for_signal_in_clickhouse_activity,
)
-from products.signals.backend.temporal.summary import SignalReportSummaryWorkflow
+from products.signals.backend.temporal.summary import RESEARCH_SANDBOX_RETRY_BUDGET, SignalReportSummaryWorkflow
from products.signals.backend.temporal.types import (
IMPLEMENTATION_DEBOUNCE_SECONDS,
RESEARCH_DEBOUNCE_SECONDS,
@@ -1716,7 +1716,8 @@ async def _process_signal_batch(
# outlive this; it's a backstop, and the report stays READY and re-promotes.
execution_timeout=timedelta(
hours=1, seconds=report_input.debounce_seconds + IMPLEMENTATION_DEBOUNCE_SECONDS
- ),
+ )
+ + RESEARCH_SANDBOX_RETRY_BUDGET,
)
except temporalio.exceptions.WorkflowAlreadyStartedError:
metrics.increment_research_run_collapsed()
diff --git a/products/signals/backend/temporal/summary.py b/products/signals/backend/temporal/summary.py
index b425bf12f65a..479a0c51dc20 100644
--- a/products/signals/backend/temporal/summary.py
+++ b/products/signals/backend/temporal/summary.py
@@ -56,6 +56,7 @@ from products.signals.backend.spend import publish_report_spend
from products.signals.backend.stack_plan import create_layer_reports, start_unblocked_layers_of_plan
from products.signals.backend.temporal import metrics
from products.signals.backend.temporal.agentic.report import (
+ RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE,
RunAgenticReportInput,
RunAgenticReportOutput,
run_agentic_report_activity,
@@ -195,6 +196,18 @@ class ReportDecision:
pending_reason: str | None = None
+RESEARCH_SANDBOX_RETRY_DELAYS = (timedelta(minutes=10), timedelta(minutes=20), timedelta(minutes=40))
+RESEARCH_SANDBOX_RETRY_BUDGET = sum(RESEARCH_SANDBOX_RETRY_DELAYS, timedelta())
+
+
+def _is_research_sandbox_unavailable(error: BaseException) -> bool:
+ cause = error.cause if isinstance(error, temporalio.exceptions.ActivityError) else error
+ return (
+ isinstance(cause, temporalio.exceptions.ApplicationError)
+ and cause.type == RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE
+ )
+
+
@temporalio.workflow.defn(name="signal-report-summary")
class SignalReportSummaryWorkflow:
"""
@@ -271,6 +284,36 @@ class SignalReportSummaryWorkflow:
retry_policy=RetryPolicy(maximum_attempts=3),
)
+ async def _run_research(
+ self, log: FilteringBoundLogger, research_input: RunAgenticReportInput
+ ) -> RunAgenticReportOutput:
+ retries = 0
+ while True:
+ try:
+ return await workflow.execute_activity(
+ run_agentic_report_activity,
+ research_input,
+ start_to_close_timeout=timedelta(hours=4),
+ heartbeat_timeout=timedelta(minutes=5),
+ # Retry once: the sandbox agent turn can time out (poll_for_turn) or hit a transient LLM 429.
+ retry_policy=RetryPolicy(maximum_attempts=2),
+ )
+ except temporalio.exceptions.ActivityError as error:
+ if (
+ not workflow.patched("signals-research-sandbox-retry")
+ or not _is_research_sandbox_unavailable(error)
+ or retries >= len(RESEARCH_SANDBOX_RETRY_DELAYS)
+ ):
+ raise
+ delay = RESEARCH_SANDBOX_RETRY_DELAYS[retries]
+ retries += 1
+ log.warning(
+ "Research sandbox unavailable, retrying after delay",
+ attempt=retries,
+ delay_seconds=int(delay.total_seconds()),
+ )
+ await workflow.sleep(delay)
+
async def _revert_report_to_candidate(self, inputs: SignalReportSummaryWorkflowInputs) -> None:
await workflow.execute_activity(
revert_report_to_candidate_activity,
@@ -480,8 +523,8 @@ class SignalReportSummaryWorkflow:
await self._revert_report_to_candidate(inputs)
return False
# 5. Run the agentic report research flow with the selected repository to use code/MCP data to assess signals
- agentic_result: RunAgenticReportOutput = await workflow.execute_activity(
- run_agentic_report_activity,
+ agentic_result = await self._run_research(
+ log,
RunAgenticReportInput(
team_id=inputs.team_id,
report_id=inputs.report_id,
@@ -489,10 +532,6 @@ class SignalReportSummaryWorkflow:
repo_selection=repo_result,
repo_selection_as_of=repo_selection_as_of,
),
- start_to_close_timeout=timedelta(hours=4),
- heartbeat_timeout=timedelta(minutes=5),
- # Retry once: the sandbox agent turn can time out (poll_for_turn) or hit a transient LLM 429.
- retry_policy=RetryPolicy(maximum_attempts=2),
)
decision = ReportDecision(
title=agentic_result.title,
@@ -699,7 +738,11 @@ class SignalReportSummaryWorkflow:
team_id=inputs.team_id,
report_id=inputs.report_id,
error=str(e),
- failure_reason="agentic_activity_error",
+ failure_reason=(
+ "research_sandbox_unavailable"
+ if _is_research_sandbox_unavailable(e)
+ else "agentic_activity_error"
+ ),
signal_count=signal_count,
source_products=source_products,
),
diff --git a/products/signals/backend/test/test_agentic_report_activity.py b/products/signals/backend/test/test_agentic_report_activity.py
index dffccaa768c0..5cabb74653b1 100644
--- a/products/signals/backend/test/test_agentic_report_activity.py
+++ b/products/signals/backend/test/test_agentic_report_activity.py
@@ -15,6 +15,7 @@ import pytest_asyncio
from asgiref.sync import sync_to_async
from parameterized import parameterized
from pydantic import ValidationError
+from temporalio.exceptions import ApplicationError
from posthog.models import Organization, Team, User
from posthog.models.organization import OrganizationMembership
@@ -63,6 +64,7 @@ from products.signals.backend.report_metrics import REPORT_METRIC_GOAL_FIELDS, R
from products.signals.backend.supersession import ImplementationResearchContext
from products.signals.backend.temporal.agentic.report import (
RESEARCH_MCP_SCOPES,
+ RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE,
RunAgenticReportInput,
_load_linked_report_context,
_load_previous_research,
@@ -84,6 +86,7 @@ from products.signals.backend.temporal.summary import (
mark_report_ready_activity,
)
from products.signals.backend.temporal.types import SignalData
+from products.tasks.backend.facade.agents import SandboxRunFailed
from products.tasks.backend.models import Task # tach-ignore
@@ -741,6 +744,48 @@ async def test_select_repository_activity_does_not_raise_with_only_user_integrat
assert captured_user_id == [user.id], "user_id should come from the UserIntegration owner"
+ "sandbox_error_type,expected_transient",
+ [("RepositoryCloneError", True), ("RepositoryAccessError", False)],
+)
+async def test_run_agentic_report_activity_flags_transient_sandbox_failures(
+ monkeypatch, ateam, sandbox_error_type, expected_transient
+):
+ report = await database_sync_to_async(SignalReport.objects.create)(
+ team=ateam,
+ status=SignalReport.Status.IN_PROGRESS,
+ signal_count=2,
+ total_weight=1.3,
+ )
+ monkeypatch.setattr("products.signals.backend.temporal.agentic.report.resolve_user_id_for_team", lambda team_id: 1)
+
+ async def fake_run_multi_turn_research(*args, **kwargs):
+ raise SandboxRunFailed("TaskRun reached terminal status=failed", error_type=sandbox_error_type)
+
+ monkeypatch.setattr(
+ "products.signals.backend.temporal.agentic.report.run_multi_turn_research", fake_run_multi_turn_research
+ )
+
+ with patch("products.signals.backend.temporal.agentic.report.Heartbeater"):
+ with pytest.raises((ApplicationError, SandboxRunFailed)) as error:
+ await run_agentic_report_activity(
+ RunAgenticReportInput(
+ team_id=ateam.id,
+ report_id=str(report.id),
+ signals=_build_signals(),
+ repo_selection=RepoSelectionResult(repository="posthog/posthog", reason="single repository"),
+ )
+ )
+
+ if expected_transient:
+ assert isinstance(error.value, ApplicationError)
+ assert error.value.type == RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE
+ else:
+ assert isinstance(error.value, SandboxRunFailed)
+
+
@pytest.mark.asyncio
@pytest.mark.django_db
async def test_run_agentic_report_activity_persists_artefacts(monkeypatch, ateam):
diff --git a/products/signals/backend/test/test_summary_workflow.py b/products/signals/backend/test/test_summary_workflow.py
index 83988d90eeb8..99fc737bed3c 100644
--- a/products/signals/backend/test/test_summary_workflow.py
+++ b/products/signals/backend/test/test_summary_workflow.py
@@ -9,7 +9,9 @@ from unittest.mock import patch
import pytest_asyncio
from asgiref.sync import sync_to_async
from temporalio import activity
+from temporalio.client import WorkflowFailureError
from temporalio.contrib.pydantic import pydantic_data_converter
+from temporalio.exceptions import ApplicationError
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
@@ -21,7 +23,11 @@ from products.signals.backend.models import SignalReport
from products.signals.backend.quota import SelfDrivingQuotaGate
from products.signals.backend.report_generation.research import ActionabilityChoice
from products.signals.backend.report_generation.select_repo import RepoSelectionResult
-from products.signals.backend.temporal.agentic.report import RunAgenticReportInput, RunAgenticReportOutput
+from products.signals.backend.temporal.agentic.report import (
+ RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE,
+ RunAgenticReportInput,
+ RunAgenticReportOutput,
+)
from products.signals.backend.temporal.agentic.select_repository import SelectRepositoryInput
from products.signals.backend.temporal.report_safety_judge import SafetyJudgeInput, SafetyJudgeOutput
from products.signals.backend.temporal.signal_queries import FetchSignalsForReportInput, FetchSignalsForReportOutput
@@ -183,6 +189,7 @@ class _Recorder:
research_metrics: list[dict[str, object]] | None = None,
checks_snapshot: dict[str, str] | None = None,
selected_repository: str | None = "owner/repo",
+ research_errors: list[Exception] | None = None,
) -> None:
self.gate_answers = gate_answers or {}
self.fetch_results = fetch_results or [[_signal_data()]]
@@ -193,6 +200,7 @@ class _Recorder:
self.checks_summary: str | None = None
self.checks_monitoring_enabled = False
self.selected_repository = selected_repository
+ self.research_errors = list(research_errors or [])
self.gate_checks: list[str] = []
self.fetches = 0
self.assigned_signal_checks = 0
@@ -275,6 +283,8 @@ async def _run_summary_workflow(recorder: _Recorder) -> None:
@activity.defn(name="run_agentic_report_activity")
async def fake_research(input: RunAgenticReportInput) -> RunAgenticReportOutput:
recorder.researches += 1
+ if recorder.research_errors:
+ raise recorder.research_errors.pop(0)
return RunAgenticReportOutput(
title="t",
summary="s",
@@ -392,6 +402,40 @@ async def test_open_gates_let_the_run_flow_through():
assert recorder.failures == 0
+def _sandbox_unavailable() -> ApplicationError:
+ return ApplicationError("sandbox gone", type=RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE)
+
+
+async def test_research_retries_after_a_sandbox_outage_instead_of_failing_the_report():
+ recorder = _Recorder(research_errors=[_sandbox_unavailable() for _ in range(2)])
+ await _run_summary_workflow(recorder)
+ assert recorder.researches == 3
+ assert recorder.failures == 0
+ assert recorder.marks_in_progress == 1
+ assert recorder.spend_publications == 1
+
+
+async def test_research_fails_the_report_once_the_sandbox_retry_budget_is_spent():
+ recorder = _Recorder(research_errors=[_sandbox_unavailable() for _ in range(8)])
+ with pytest.raises(WorkflowFailureError):
+ await _run_summary_workflow(recorder)
+ assert recorder.researches == 8
+ assert recorder.failures == 1
+ assert recorder.failure_reasons == ["research_sandbox_unavailable"]
+
+
+async def test_research_does_not_retry_other_activity_errors():
+ recorder = _Recorder(research_errors=[ApplicationError("bad output", type="ValueError") for _ in range(2)])
+ with pytest.raises(WorkflowFailureError):
+ await _run_summary_workflow(recorder)
+ assert recorder.researches == 2
+ assert recorder.failures == 1
+ assert recorder.failure_reasons == ["agentic_activity_error"]
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"choice,target",
diff --git a/products/tasks/backend/exceptions.py b/products/tasks/backend/exceptions.py
index b60c9740ef0c..59dedfd6bb2e 100644
--- a/products/tasks/backend/exceptions.py
+++ b/products/tasks/backend/exceptions.py
@@ -12,6 +12,7 @@ from products.tasks.backend.facade.compute_quota import ComputeBillingLimitExcee
SANDBOX_RATE_LIMIT_BASE_DELAY_SECONDS = 45
SANDBOX_RATE_LIMIT_MAX_DELAY_SECONDS = 300
+TEMPORAL_TIMEOUT_ERROR_TYPES = frozenset({"start_to_close", "schedule_to_start", "schedule_to_close", "heartbeat"})
def sandbox_rate_limit_retry_delay(attempt: int) -> float:
@@ -54,6 +55,18 @@ class ProcessTaskTransientError(ProcessTaskError):
super().__init__(message, context, cause, non_retryable=False, **kwargs)
+def is_transient_failure_error_type(error_type: str | None) -> bool:
+ if error_type in TEMPORAL_TIMEOUT_ERROR_TYPES:
+ return True
+ pending = [ProcessTaskTransientError]
+ while pending:
+ error_class = pending.pop()
+ if error_class.__name__ == error_type:
+ return True
+ pending.extend(error_class.__subclasses__())
+ return False
+
+
class TaskNotFoundError(ProcessTaskFatalError):
pass
diff --git a/products/tasks/backend/facade/agents.py b/products/tasks/backend/facade/agents.py
index 0fdf65075f84..06677ee5406e 100644
--- a/products/tasks/backend/facade/agents.py
+++ b/products/tasks/backend/facade/agents.py
@@ -14,6 +14,7 @@ from products.tasks.backend.logic.services.custom_prompt_internals import (
CustomPromptSandboxContext,
EmptyAgentTurnError,
OutputFn,
+ SandboxRunFailed,
TruncatedAgentOutputError,
TurnPollResult,
TurnPollTimeout,
@@ -57,6 +58,7 @@ __all__ = [
"LocalSkillsCache",
"MultiTurnSession",
"OutputFn",
+ "SandboxRunFailed",
"TruncatedAgentOutputError",
"TurnPollResult",
"TurnPollTimeout",
diff --git a/products/tasks/backend/logic/services/custom_prompt_internals.py b/products/tasks/backend/logic/services/custom_prompt_internals.py
index 8dcd8083229b..9cfc7c6947fb 100644
--- a/products/tasks/backend/logic/services/custom_prompt_internals.py
+++ b/products/tasks/backend/logic/services/custom_prompt_internals.py
@@ -22,6 +22,7 @@ from posthog.storage import object_storage
from posthog.storage.object_storage import ObjectStorageError
from posthog.temporal.oauth import PosthogMcpScopes
+from products.tasks.backend.exceptions import is_transient_failure_error_type
from products.tasks.backend.facade.api import upload_task_run_artifacts
from products.tasks.backend.facade.contracts import AgentTaskRunDTO, SessionFile
from products.tasks.backend.models import MCPBuiltInAgentKey, Task, TaskRun, stamp_pending_user_message_id
@@ -269,6 +270,16 @@ class AgentTurnFailed(RuntimeError):
return self.category in UPSTREAM_RETRYABLE_ERROR_CATEGORIES
+class SandboxRunFailed(RuntimeError):
+ def __init__(self, message: str, *, error_type: str | None) -> None:
+ super().__init__(message)
+ self.error_type = error_type
+
+ @property
+ def transient(self) -> bool:
+ return is_transient_failure_error_type(self.error_type)
+
+
SESSION_FILE_ARTIFACT_TYPE = "context"
@@ -663,6 +674,7 @@ async def poll_for_turn(
task_run,
refreshed_status=refreshed.status,
error_message=refreshed.error_message,
+ failure_error_type=(refreshed.state or {}).get("failure_error_type"),
printed_lines=printed_lines,
original_skip_lines=original_skip_lines,
verbose=verbose,
@@ -688,6 +700,7 @@ async def poll_for_turn(
task_run,
refreshed_status=refreshed.status,
error_message=refreshed.error_message,
+ failure_error_type=(refreshed.state or {}).get("failure_error_type"),
printed_lines=printed_lines,
original_skip_lines=original_skip_lines,
verbose=verbose,
@@ -870,6 +883,7 @@ async def _drain_final_log(
*,
refreshed_status: str,
error_message: str | None = None,
+ failure_error_type: str | None = None,
printed_lines: int,
original_skip_lines: int,
verbose: bool,
@@ -912,9 +926,10 @@ async def _drain_final_log(
)
reason = "end_turn with empty response" if final_state.empty_end_turn else "no agent message"
cause = f" (cause: {error_message})" if error_message else ""
- raise RuntimeError(
- f"custom_prompt - drain_final_log: TaskRun reached terminal status={refreshed_status}{cause} — {reason}"
- )
+ message = f"custom_prompt - drain_final_log: TaskRun reached terminal status={refreshed_status}{cause} — {reason}"
+ if refreshed_status == TaskRun.Status.FAILED:
+ raise SandboxRunFailed(message, error_type=failure_error_type)
+ raise RuntimeError(message)
def _extract_agent_error(log_content: str | None, skip_lines: int = 0) -> AgentError | None:
diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py
index cbf11d719246..035c5569cc43 100644
--- a/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py
+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py
@@ -460,6 +460,8 @@ class TestUpdateTaskRunStatusActivity:
assert len(props["error_message"]) == 500
assert props["error_message"].endswith("TypeError: cannot read boot manifest")
assert props.get("sandbox_backend") == expected_backend
+ test_task_run.refresh_from_db()
+ assert test_task_run.state.get("failure_error_type") == (expected_error_type if error_type else None)
@pytest.mark.django_db(transaction=True)
@patch("products.tasks.backend.models.posthoganalytics.capture")
diff --git a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
index ffbe1147acbd..95cc62162c64 100644
--- a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
+++ b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
@@ -109,6 +109,14 @@ def update_task_run_status(input: UpdateTaskRunStatusInput) -> None:
if marker in _TERMINAL_STATE_MARKERS:
# Atomic merge so concurrent state writers aren't clobbered; reassigned so reads below see it.
task_run.state = TaskRun.update_state_atomic(task_run.id, updates={marker: True})
+ if (
+ input.status == TaskRun.Status.FAILED
+ and input.error_type
+ and not (task_run.state or {}).get("failure_error_type")
+ ):
+ task_run.state = TaskRun.update_state_atomic(
+ task_run.id, updates={"failure_error_type": input.error_type}
+ )
if not task_run.completed_at and (
input.status in [TaskRun.Status.COMPLETED, TaskRun.Status.FAILED]
or (input.status == TaskRun.Status.CANCELLED and task_run.environment == TaskRun.Environment.CLOUD)
diff --git a/products/tasks/backend/tests/agent_log_fixtures.py b/products/tasks/backend/tests/agent_log_fixtures.py
index 5942ae92db5f..5f9bff637de7 100644
--- a/products/tasks/backend/tests/agent_log_fixtures.py
+++ b/products/tasks/backend/tests/agent_log_fixtures.py
@@ -167,3 +167,4 @@ class FakeTaskRun:
log_url: str = "s3://fake/log"
status: str = "running"
error_message: str | None = None
+ state: dict[str, object] | None = None
diff --git a/products/tasks/backend/tests/test_exceptions.py b/products/tasks/backend/tests/test_exceptions.py
index 53b865a3927f..7c9323007c17 100644
--- a/products/tasks/backend/tests/test_exceptions.py
+++ b/products/tasks/backend/tests/test_exceptions.py
@@ -14,6 +14,7 @@ from products.tasks.backend.exceptions import (
SandboxControlPlaneUnavailableError,
SandboxNotRunningError,
SandboxRateLimitedError,
+ is_transient_failure_error_type,
)
@@ -60,3 +61,19 @@ def test_rate_limit_backoff_follows_the_activity_attempt(attempt: int, expected_
assert error.value.non_retryable is False
assert error.value.next_retry_delay is not None
assert timedelta(seconds=expected_floor) <= error.value.next_retry_delay <= timedelta(seconds=expected_ceiling)
+
+
+ "error_type,expected",
+ [
+ ("RepositoryCloneError", True),
+ ("SandboxRateLimitedError", True),
+ ("start_to_close", True),
+ ("RepositoryAccessError", False),
+ ("CredentialUnavailableError", False),
+ ("unspecified", False),
+ (None, False),
+ ],
+)
+def test_transient_failure_error_types_cover_transient_subclasses_and_timeouts(error_type: str | None, expected: bool):
+ assert is_transient_failure_error_type(error_type) is expected
diff --git a/products/tasks/backend/tests/test_multi_turn_session.py b/products/tasks/backend/tests/test_multi_turn_session.py
index 671ea159475c..3bf3d3919f96 100644
--- a/products/tasks/backend/tests/test_multi_turn_session.py
+++ b/products/tasks/backend/tests/test_multi_turn_session.py
@@ -26,6 +26,7 @@ from products.tasks.backend.logic.services.custom_prompt_internals import (
AgentTurnFailed,
CustomPromptSandboxContext,
EmptyAgentTurnError,
+ SandboxRunFailed,
TurnPollResult,
TurnPollTimeout,
_create_task_and_trigger,
@@ -1227,11 +1228,21 @@ class TestPollForTurnSurfacesAgentError:
assert exc_info.value.retryable_upstream is False
@pytest.mark.asyncio
- async def test_missing_structured_error_keeps_generic_behavior(self):
+ @pytest.mark.parametrize(
+ "state,expected_error_type,expected_transient",
+ [
+ (None, None, False),
+ ({"failure_error_type": "RepositoryCloneError"}, "RepositoryCloneError", True),
+ ({"failure_error_type": "RepositoryAccessError"}, "RepositoryAccessError", False),
+ ],
+ )
+ async def test_missing_structured_error_keeps_generic_behavior(
+ self, state, expected_error_type, expected_transient
+ ):
# No _posthog/error line: the drain must keep today's generic message and
# must not touch TaskRun.error_message.
turn = [_user_message_line("summarize"), _usage_update_line(0)]
- fake_task_run = FakeTaskRun(status="failed", error_message="Activity task failed")
+ fake_task_run = FakeTaskRun(status="failed", error_message="Activity task failed", state=state)
persist = AsyncMock()
with (
@@ -1250,6 +1261,9 @@ class TestPollForTurnSurfacesAgentError:
assert "Activity task failed" in str(exc_info.value)
# No agent classification to carry, so the generic failure stays untyped.
assert not isinstance(exc_info.value, AgentTurnFailed)
+ assert isinstance(exc_info.value, SandboxRunFailed)
+ assert exc_info.value.error_type == expected_error_type
+ assert exc_info.value.transient is expected_transient
persist.assert_not_awaited()
assert fake_task_run.error_message == "Activity task failed"