:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signals): retry research after a transient sandbox failure (#123983)
Alessandro Pogliaghie5c8eb532c0c
diff --git a/products/signals/backend/management/commands/rerun_signal_report.py b/products/signals/backend/management/commands/rerun_signal_report.pyindex 029ba4616a83..e0e0a23b988b 100644--- a/products/signals/backend/management/commands/rerun_signal_report.py+++ b/products/signals/backend/management/commands/rerun_signal_report.py@@ -25,7 +25,7 @@ from products.signals.backend.models import SignalReportfrom products.signals.backend.quota import self_driving_quota_gatefrom products.signals.backend.supersession import pending_replacementfrom products.signals.backend.task_run_artefacts import SIGNALS_PRODUCT, TASK_RUN_TYPE_IMPLEMENTATION-from products.signals.backend.temporal.summary import SignalReportSummaryWorkflow+from products.signals.backend.temporal.summary import RESEARCH_SANDBOX_RETRY_BUDGET, SignalReportSummaryWorkflowfrom products.signals.backend.temporal.types import IMPLEMENTATION_DEBOUNCE_SECONDS, SignalReportSummaryWorkflowInputsfrom products.tasks.backend.facade import api as tasks_facadefrom products.tasks.backend.facade.usage import task_run_usage_limited@@ -74,7 +74,8 @@ class Command(BaseCommand):id=SignalReportSummaryWorkflow.workflow_id_for(inputs.team_id, inputs.report_id),task_queue=settings.VIDEO_EXPORT_TASK_QUEUE,id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE,- execution_timeout=timedelta(hours=1, seconds=IMPLEMENTATION_DEBOUNCE_SECONDS),+ execution_timeout=timedelta(hours=1, seconds=IMPLEMENTATION_DEBOUNCE_SECONDS)+ + RESEARCH_SANDBOX_RETRY_BUDGET,)def handle(self, *args: object, **options: object) -> None:diff --git a/products/signals/backend/temporal/agentic/report.py b/products/signals/backend/temporal/agentic/report.pyindex c4fbd5560cf6..03f191dabfe0 100644--- a/products/signals/backend/temporal/agentic/report.py+++ b/products/signals/backend/temporal/agentic/report.py@@ -84,7 +84,7 @@ from products.signals.backend.typed_report_links import (outgoing_links,)from products.tasks.backend.facade import api as tasks_facade-from products.tasks.backend.facade.agents import CustomPromptSandboxContext+from products.tasks.backend.facade.agents import CustomPromptSandboxContext, SandboxRunFailedlogger = structlog.get_logger(__name__)@@ -1097,6 +1097,9 @@ def _capture_research_steering_attached(*, team_id: int, report_id: str, steerinlogger.exception("Failed to capture signals_research_steering_attached", report_id=report_id)+RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE = "ResearchSandboxUnavailable"++@temporalio.activity.defn@scoped_temporal()@close_db_connections@@ -1271,6 +1274,19 @@ async def run_agentic_report_activity(input: RunAgenticReportInput) -> RunAgentiresearch_task_id=result.research_task_id,charts_enabled=True,)+ except SandboxRunFailed as error:+ logger.exception(+ "signals agentic report failed",+ report_id=input.report_id,+ team_id=input.team_id,+ error=str(error),+ sandbox_error_type=error.error_type,+ )+ if error.transient:+ raise temporalio.exceptions.ApplicationError(+ str(error), type=RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE+ ) from error+ raiseexcept Exception as error:logger.exception("signals agentic report failed",diff --git a/products/signals/backend/temporal/grouping.py b/products/signals/backend/temporal/grouping.pyindex 61b5c53af9e4..18f3f81f7130 100644--- a/products/signals/backend/temporal/grouping.py+++ b/products/signals/backend/temporal/grouping.py@@ -66,7 +66,7 @@ from products.signals.backend.temporal.signal_queries import (soft_delete_report_signals,wait_for_signal_in_clickhouse_activity,)-from products.signals.backend.temporal.summary import SignalReportSummaryWorkflow+from products.signals.backend.temporal.summary import RESEARCH_SANDBOX_RETRY_BUDGET, SignalReportSummaryWorkflowfrom products.signals.backend.temporal.types import (IMPLEMENTATION_DEBOUNCE_SECONDS,RESEARCH_DEBOUNCE_SECONDS,@@ -1716,7 +1716,8 @@ async def _process_signal_batch(# outlive this; it's a backstop, and the report stays READY and re-promotes.execution_timeout=timedelta(hours=1, seconds=report_input.debounce_seconds + IMPLEMENTATION_DEBOUNCE_SECONDS- ),+ )+ + RESEARCH_SANDBOX_RETRY_BUDGET,)except temporalio.exceptions.WorkflowAlreadyStartedError:metrics.increment_research_run_collapsed()diff --git a/products/signals/backend/temporal/summary.py b/products/signals/backend/temporal/summary.pyindex b425bf12f65a..479a0c51dc20 100644--- a/products/signals/backend/temporal/summary.py+++ b/products/signals/backend/temporal/summary.py@@ -56,6 +56,7 @@ from products.signals.backend.spend import publish_report_spendfrom products.signals.backend.stack_plan import create_layer_reports, start_unblocked_layers_of_planfrom products.signals.backend.temporal import metricsfrom products.signals.backend.temporal.agentic.report import (+ RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE,RunAgenticReportInput,RunAgenticReportOutput,run_agentic_report_activity,@@ -195,6 +196,18 @@ class ReportDecision:pending_reason: str | None = None+RESEARCH_SANDBOX_RETRY_DELAYS = (timedelta(minutes=10), timedelta(minutes=20), timedelta(minutes=40))+RESEARCH_SANDBOX_RETRY_BUDGET = sum(RESEARCH_SANDBOX_RETRY_DELAYS, timedelta())+++def _is_research_sandbox_unavailable(error: BaseException) -> bool:+ cause = error.cause if isinstance(error, temporalio.exceptions.ActivityError) else error+ return (+ isinstance(cause, temporalio.exceptions.ApplicationError)+ and cause.type == RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE+ )++@temporalio.workflow.defn(name="signal-report-summary")class SignalReportSummaryWorkflow:"""@@ -271,6 +284,36 @@ class SignalReportSummaryWorkflow:retry_policy=RetryPolicy(maximum_attempts=3),)+ async def _run_research(+ self, log: FilteringBoundLogger, research_input: RunAgenticReportInput+ ) -> RunAgenticReportOutput:+ retries = 0+ while True:+ try:+ return await workflow.execute_activity(+ run_agentic_report_activity,+ research_input,+ start_to_close_timeout=timedelta(hours=4),+ heartbeat_timeout=timedelta(minutes=5),+ # Retry once: the sandbox agent turn can time out (poll_for_turn) or hit a transient LLM 429.+ retry_policy=RetryPolicy(maximum_attempts=2),+ )+ except temporalio.exceptions.ActivityError as error:+ if (+ not workflow.patched("signals-research-sandbox-retry")+ or not _is_research_sandbox_unavailable(error)+ or retries >= len(RESEARCH_SANDBOX_RETRY_DELAYS)+ ):+ raise+ delay = RESEARCH_SANDBOX_RETRY_DELAYS[retries]+ retries += 1+ log.warning(+ "Research sandbox unavailable, retrying after delay",+ attempt=retries,+ delay_seconds=int(delay.total_seconds()),+ )+ await workflow.sleep(delay)+async def _revert_report_to_candidate(self, inputs: SignalReportSummaryWorkflowInputs) -> None:await workflow.execute_activity(revert_report_to_candidate_activity,@@ -480,8 +523,8 @@ class SignalReportSummaryWorkflow:await self._revert_report_to_candidate(inputs)return False# 5. Run the agentic report research flow with the selected repository to use code/MCP data to assess signals- agentic_result: RunAgenticReportOutput = await workflow.execute_activity(- run_agentic_report_activity,+ agentic_result = await self._run_research(+ log,RunAgenticReportInput(team_id=inputs.team_id,report_id=inputs.report_id,@@ -489,10 +532,6 @@ class SignalReportSummaryWorkflow:repo_selection=repo_result,repo_selection_as_of=repo_selection_as_of,),- start_to_close_timeout=timedelta(hours=4),- heartbeat_timeout=timedelta(minutes=5),- # Retry once: the sandbox agent turn can time out (poll_for_turn) or hit a transient LLM 429.- retry_policy=RetryPolicy(maximum_attempts=2),)decision = ReportDecision(title=agentic_result.title,@@ -699,7 +738,11 @@ class SignalReportSummaryWorkflow:team_id=inputs.team_id,report_id=inputs.report_id,error=str(e),- failure_reason="agentic_activity_error",+ failure_reason=(+ "research_sandbox_unavailable"+ if _is_research_sandbox_unavailable(e)+ else "agentic_activity_error"+ ),signal_count=signal_count,source_products=source_products,),diff --git a/products/signals/backend/test/test_agentic_report_activity.py b/products/signals/backend/test/test_agentic_report_activity.pyindex dffccaa768c0..5cabb74653b1 100644--- a/products/signals/backend/test/test_agentic_report_activity.py+++ b/products/signals/backend/test/test_agentic_report_activity.py@@ -15,6 +15,7 @@ import pytest_asynciofrom asgiref.sync import sync_to_asyncfrom parameterized import parameterizedfrom pydantic import ValidationError+from temporalio.exceptions import ApplicationErrorfrom posthog.models import Organization, Team, Userfrom posthog.models.organization import OrganizationMembership@@ -63,6 +64,7 @@ from products.signals.backend.report_metrics import REPORT_METRIC_GOAL_FIELDS, Rfrom products.signals.backend.supersession import ImplementationResearchContextfrom products.signals.backend.temporal.agentic.report import (RESEARCH_MCP_SCOPES,+ RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE,RunAgenticReportInput,_load_linked_report_context,_load_previous_research,@@ -84,6 +86,7 @@ from products.signals.backend.temporal.summary import (mark_report_ready_activity,)from products.signals.backend.temporal.types import SignalData+from products.tasks.backend.facade.agents import SandboxRunFailedfrom products.tasks.backend.models import Task # tach-ignore@@ -741,6 +744,48 @@ async def test_select_repository_activity_does_not_raise_with_only_user_integratassert captured_user_id == [user.id], "user_id should come from the UserIntegration owner"+ "sandbox_error_type,expected_transient",+ [("RepositoryCloneError", True), ("RepositoryAccessError", False)],+)+async def test_run_agentic_report_activity_flags_transient_sandbox_failures(+ monkeypatch, ateam, sandbox_error_type, expected_transient+):+ report = await database_sync_to_async(SignalReport.objects.create)(+ team=ateam,+ status=SignalReport.Status.IN_PROGRESS,+ signal_count=2,+ total_weight=1.3,+ )+ monkeypatch.setattr("products.signals.backend.temporal.agentic.report.resolve_user_id_for_team", lambda team_id: 1)++ async def fake_run_multi_turn_research(*args, **kwargs):+ raise SandboxRunFailed("TaskRun reached terminal status=failed", error_type=sandbox_error_type)++ monkeypatch.setattr(+ "products.signals.backend.temporal.agentic.report.run_multi_turn_research", fake_run_multi_turn_research+ )++ with patch("products.signals.backend.temporal.agentic.report.Heartbeater"):+ with pytest.raises((ApplicationError, SandboxRunFailed)) as error:+ await run_agentic_report_activity(+ RunAgenticReportInput(+ team_id=ateam.id,+ report_id=str(report.id),+ signals=_build_signals(),+ repo_selection=RepoSelectionResult(repository="posthog/posthog", reason="single repository"),+ )+ )++ if expected_transient:+ assert isinstance(error.value, ApplicationError)+ assert error.value.type == RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE+ else:+ assert isinstance(error.value, SandboxRunFailed)++@pytest.mark.asyncio@pytest.mark.django_dbasync def test_run_agentic_report_activity_persists_artefacts(monkeypatch, ateam):diff --git a/products/signals/backend/test/test_summary_workflow.py b/products/signals/backend/test/test_summary_workflow.pyindex 83988d90eeb8..99fc737bed3c 100644--- a/products/signals/backend/test/test_summary_workflow.py+++ b/products/signals/backend/test/test_summary_workflow.py@@ -9,7 +9,9 @@ from unittest.mock import patchimport pytest_asynciofrom asgiref.sync import sync_to_asyncfrom temporalio import activity+from temporalio.client import WorkflowFailureErrorfrom temporalio.contrib.pydantic import pydantic_data_converter+from temporalio.exceptions import ApplicationErrorfrom temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Worker@@ -21,7 +23,11 @@ from products.signals.backend.models import SignalReportfrom products.signals.backend.quota import SelfDrivingQuotaGatefrom products.signals.backend.report_generation.research import ActionabilityChoicefrom products.signals.backend.report_generation.select_repo import RepoSelectionResult-from products.signals.backend.temporal.agentic.report import RunAgenticReportInput, RunAgenticReportOutput+from products.signals.backend.temporal.agentic.report import (+ RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE,+ RunAgenticReportInput,+ RunAgenticReportOutput,+)from products.signals.backend.temporal.agentic.select_repository import SelectRepositoryInputfrom products.signals.backend.temporal.report_safety_judge import SafetyJudgeInput, SafetyJudgeOutputfrom products.signals.backend.temporal.signal_queries import FetchSignalsForReportInput, FetchSignalsForReportOutput@@ -183,6 +189,7 @@ class _Recorder:research_metrics: list[dict[str, object]] | None = None,checks_snapshot: dict[str, str] | None = None,selected_repository: str | None = "owner/repo",+ research_errors: list[Exception] | None = None,) -> None:self.gate_answers = gate_answers or {}self.fetch_results = fetch_results or [[_signal_data()]]@@ -193,6 +200,7 @@ class _Recorder:self.checks_summary: str | None = Noneself.checks_monitoring_enabled = Falseself.selected_repository = selected_repository+ self.research_errors = list(research_errors or [])self.gate_checks: list[str] = []self.fetches = 0self.assigned_signal_checks = 0@@ -275,6 +283,8 @@ async def _run_summary_workflow(recorder: _Recorder) -> None:@activity.defn(name="run_agentic_report_activity")async def fake_research(input: RunAgenticReportInput) -> RunAgenticReportOutput:recorder.researches += 1+ if recorder.research_errors:+ raise recorder.research_errors.pop(0)return RunAgenticReportOutput(title="t",summary="s",@@ -392,6 +402,40 @@ async def test_open_gates_let_the_run_flow_through():assert recorder.failures == 0+def _sandbox_unavailable() -> ApplicationError:+ return ApplicationError("sandbox gone", type=RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE)+++async def test_research_retries_after_a_sandbox_outage_instead_of_failing_the_report():+ recorder = _Recorder(research_errors=[_sandbox_unavailable() for _ in range(2)])+ await _run_summary_workflow(recorder)+ assert recorder.researches == 3+ assert recorder.failures == 0+ assert recorder.marks_in_progress == 1+ assert recorder.spend_publications == 1+++async def test_research_fails_the_report_once_the_sandbox_retry_budget_is_spent():+ recorder = _Recorder(research_errors=[_sandbox_unavailable() for _ in range(8)])+ with pytest.raises(WorkflowFailureError):+ await _run_summary_workflow(recorder)+ assert recorder.researches == 8+ assert recorder.failures == 1+ assert recorder.failure_reasons == ["research_sandbox_unavailable"]+++async def test_research_does_not_retry_other_activity_errors():+ recorder = _Recorder(research_errors=[ApplicationError("bad output", type="ValueError") for _ in range(2)])+ with pytest.raises(WorkflowFailureError):+ await _run_summary_workflow(recorder)+ assert recorder.researches == 2+ assert recorder.failures == 1+ assert recorder.failure_reasons == ["agentic_activity_error"]++@pytest.mark.asyncio@pytest.mark.parametrize("choice,target",diff --git a/products/tasks/backend/exceptions.py b/products/tasks/backend/exceptions.pyindex b60c9740ef0c..59dedfd6bb2e 100644--- a/products/tasks/backend/exceptions.py+++ b/products/tasks/backend/exceptions.py@@ -12,6 +12,7 @@ from products.tasks.backend.facade.compute_quota import ComputeBillingLimitExceeSANDBOX_RATE_LIMIT_BASE_DELAY_SECONDS = 45SANDBOX_RATE_LIMIT_MAX_DELAY_SECONDS = 300+TEMPORAL_TIMEOUT_ERROR_TYPES = frozenset({"start_to_close", "schedule_to_start", "schedule_to_close", "heartbeat"})def sandbox_rate_limit_retry_delay(attempt: int) -> float:@@ -54,6 +55,18 @@ class ProcessTaskTransientError(ProcessTaskError):super().__init__(message, context, cause, non_retryable=False, **kwargs)+def is_transient_failure_error_type(error_type: str | None) -> bool:+ if error_type in TEMPORAL_TIMEOUT_ERROR_TYPES:+ return True+ pending = [ProcessTaskTransientError]+ while pending:+ error_class = pending.pop()+ if error_class.__name__ == error_type:+ return True+ pending.extend(error_class.__subclasses__())+ return False++class TaskNotFoundError(ProcessTaskFatalError):passdiff --git a/products/tasks/backend/facade/agents.py b/products/tasks/backend/facade/agents.pyindex 0fdf65075f84..06677ee5406e 100644--- a/products/tasks/backend/facade/agents.py+++ b/products/tasks/backend/facade/agents.py@@ -14,6 +14,7 @@ from products.tasks.backend.logic.services.custom_prompt_internals import (CustomPromptSandboxContext,EmptyAgentTurnError,OutputFn,+ SandboxRunFailed,TruncatedAgentOutputError,TurnPollResult,TurnPollTimeout,@@ -57,6 +58,7 @@ __all__ = ["LocalSkillsCache","MultiTurnSession","OutputFn",+ "SandboxRunFailed","TruncatedAgentOutputError","TurnPollResult","TurnPollTimeout",diff --git a/products/tasks/backend/logic/services/custom_prompt_internals.py b/products/tasks/backend/logic/services/custom_prompt_internals.pyindex 8dcd8083229b..9cfc7c6947fb 100644--- a/products/tasks/backend/logic/services/custom_prompt_internals.py+++ b/products/tasks/backend/logic/services/custom_prompt_internals.py@@ -22,6 +22,7 @@ from posthog.storage import object_storagefrom posthog.storage.object_storage import ObjectStorageErrorfrom posthog.temporal.oauth import PosthogMcpScopes+from products.tasks.backend.exceptions import is_transient_failure_error_typefrom products.tasks.backend.facade.api import upload_task_run_artifactsfrom products.tasks.backend.facade.contracts import AgentTaskRunDTO, SessionFilefrom products.tasks.backend.models import MCPBuiltInAgentKey, Task, TaskRun, stamp_pending_user_message_id@@ -269,6 +270,16 @@ class AgentTurnFailed(RuntimeError):return self.category in UPSTREAM_RETRYABLE_ERROR_CATEGORIES+class SandboxRunFailed(RuntimeError):+ def __init__(self, message: str, *, error_type: str | None) -> None:+ super().__init__(message)+ self.error_type = error_type++ @property+ def transient(self) -> bool:+ return is_transient_failure_error_type(self.error_type)++SESSION_FILE_ARTIFACT_TYPE = "context"@@ -663,6 +674,7 @@ async def poll_for_turn(task_run,refreshed_status=refreshed.status,error_message=refreshed.error_message,+ failure_error_type=(refreshed.state or {}).get("failure_error_type"),printed_lines=printed_lines,original_skip_lines=original_skip_lines,verbose=verbose,@@ -688,6 +700,7 @@ async def poll_for_turn(task_run,refreshed_status=refreshed.status,error_message=refreshed.error_message,+ failure_error_type=(refreshed.state or {}).get("failure_error_type"),printed_lines=printed_lines,original_skip_lines=original_skip_lines,verbose=verbose,@@ -870,6 +883,7 @@ async def _drain_final_log(*,refreshed_status: str,error_message: str | None = None,+ failure_error_type: str | None = None,printed_lines: int,original_skip_lines: int,verbose: bool,@@ -912,9 +926,10 @@ async def _drain_final_log()reason = "end_turn with empty response" if final_state.empty_end_turn else "no agent message"cause = f" (cause: {error_message})" if error_message else ""- raise RuntimeError(- f"custom_prompt - drain_final_log: TaskRun reached terminal status={refreshed_status}{cause} — {reason}"- )+ message = f"custom_prompt - drain_final_log: TaskRun reached terminal status={refreshed_status}{cause} — {reason}"+ if refreshed_status == TaskRun.Status.FAILED:+ raise SandboxRunFailed(message, error_type=failure_error_type)+ raise RuntimeError(message)def _extract_agent_error(log_content: str | None, skip_lines: int = 0) -> AgentError | None:diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.pyindex cbf11d719246..035c5569cc43 100644--- a/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py@@ -460,6 +460,8 @@ class TestUpdateTaskRunStatusActivity:assert len(props["error_message"]) == 500assert props["error_message"].endswith("TypeError: cannot read boot manifest")assert props.get("sandbox_backend") == expected_backend+ test_task_run.refresh_from_db()+ assert test_task_run.state.get("failure_error_type") == (expected_error_type if error_type else None)@pytest.mark.django_db(transaction=True)@patch("products.tasks.backend.models.posthoganalytics.capture")diff --git a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.pyindex ffbe1147acbd..95cc62162c64 100644--- a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py+++ b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py@@ -109,6 +109,14 @@ def update_task_run_status(input: UpdateTaskRunStatusInput) -> None:if marker in _TERMINAL_STATE_MARKERS:# Atomic merge so concurrent state writers aren't clobbered; reassigned so reads below see it.task_run.state = TaskRun.update_state_atomic(task_run.id, updates={marker: True})+ if (+ input.status == TaskRun.Status.FAILED+ and input.error_type+ and not (task_run.state or {}).get("failure_error_type")+ ):+ task_run.state = TaskRun.update_state_atomic(+ task_run.id, updates={"failure_error_type": input.error_type}+ )if not task_run.completed_at and (input.status in [TaskRun.Status.COMPLETED, TaskRun.Status.FAILED]or (input.status == TaskRun.Status.CANCELLED and task_run.environment == TaskRun.Environment.CLOUD)diff --git a/products/tasks/backend/tests/agent_log_fixtures.py b/products/tasks/backend/tests/agent_log_fixtures.pyindex 5942ae92db5f..5f9bff637de7 100644--- a/products/tasks/backend/tests/agent_log_fixtures.py+++ b/products/tasks/backend/tests/agent_log_fixtures.py@@ -167,3 +167,4 @@ class FakeTaskRun:log_url: str = "s3://fake/log"status: str = "running"error_message: str | None = None+ state: dict[str, object] | None = Nonediff --git a/products/tasks/backend/tests/test_exceptions.py b/products/tasks/backend/tests/test_exceptions.pyindex 53b865a3927f..7c9323007c17 100644--- a/products/tasks/backend/tests/test_exceptions.py+++ b/products/tasks/backend/tests/test_exceptions.py@@ -14,6 +14,7 @@ from products.tasks.backend.exceptions import (SandboxControlPlaneUnavailableError,SandboxNotRunningError,SandboxRateLimitedError,+ is_transient_failure_error_type,)@@ -60,3 +61,19 @@ def test_rate_limit_backoff_follows_the_activity_attempt(attempt: int, expected_assert error.value.non_retryable is Falseassert error.value.next_retry_delay is not Noneassert timedelta(seconds=expected_floor) <= error.value.next_retry_delay <= timedelta(seconds=expected_ceiling)+++ "error_type,expected",+ [+ ("RepositoryCloneError", True),+ ("SandboxRateLimitedError", True),+ ("start_to_close", True),+ ("RepositoryAccessError", False),+ ("CredentialUnavailableError", False),+ ("unspecified", False),+ (None, False),+ ],+)+def test_transient_failure_error_types_cover_transient_subclasses_and_timeouts(error_type: str | None, expected: bool):+ assert is_transient_failure_error_type(error_type) is expecteddiff --git a/products/tasks/backend/tests/test_multi_turn_session.py b/products/tasks/backend/tests/test_multi_turn_session.pyindex 671ea159475c..3bf3d3919f96 100644--- a/products/tasks/backend/tests/test_multi_turn_session.py+++ b/products/tasks/backend/tests/test_multi_turn_session.py@@ -26,6 +26,7 @@ from products.tasks.backend.logic.services.custom_prompt_internals import (AgentTurnFailed,CustomPromptSandboxContext,EmptyAgentTurnError,+ SandboxRunFailed,TurnPollResult,TurnPollTimeout,_create_task_and_trigger,@@ -1227,11 +1228,21 @@ class TestPollForTurnSurfacesAgentError:assert exc_info.value.retryable_upstream is False@pytest.mark.asyncio- async def test_missing_structured_error_keeps_generic_behavior(self):+ @pytest.mark.parametrize(+ "state,expected_error_type,expected_transient",+ [+ (None, None, False),+ ({"failure_error_type": "RepositoryCloneError"}, "RepositoryCloneError", True),+ ({"failure_error_type": "RepositoryAccessError"}, "RepositoryAccessError", False),+ ],+ )+ async def test_missing_structured_error_keeps_generic_behavior(+ self, state, expected_error_type, expected_transient+ ):# No _posthog/error line: the drain must keep today's generic message and# must not touch TaskRun.error_message.turn = [_user_message_line("summarize"), _usage_update_line(0)]- fake_task_run = FakeTaskRun(status="failed", error_message="Activity task failed")+ fake_task_run = FakeTaskRun(status="failed", error_message="Activity task failed", state=state)persist = AsyncMock()with (@@ -1250,6 +1261,9 @@ class TestPollForTurnSurfacesAgentError:assert "Activity task failed" in str(exc_info.value)# No agent classification to carry, so the generic failure stays untyped.assert not isinstance(exc_info.value, AgentTurnFailed)+ assert isinstance(exc_info.value, SandboxRunFailed)+ assert exc_info.value.error_type == expected_error_type+ assert exc_info.value.transient is expected_transientpersist.assert_not_awaited()assert fake_task_run.error_message == "Activity task failed"