Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): explain and notify research startup failures (#124666)

Alessandro Pogliaghiefcc2b71760b
diff --git a/docs/internal/task-run-api.md b/docs/internal/task-run-api.md
index e749b133285a..79f6f95190f1 100644
--- a/docs/internal/task-run-api.md
+++ b/docs/internal/task-run-api.md
@@ -36,6 +36,7 @@ Temporary Codex credential-service outages remain retryable. Missing or invalid
The worker records `agent_startup_attempt` immediately before a real launch and `agent_startup_success` after readiness succeeds.
Successful initialization clears the startup failure and clears the crash message while the run is non-terminal. A terminal run keeps its final error. A later failed attempt preserves the last completed startup as recovery evidence.
These startup state fields cannot be written, appended to, or removed through the run PATCH API, including by sandbox clients.
+Recovery notifications bind the repository, provider, runtime, and startup time. Reusing an already healthy server preserves its original startup evidence.
Older agents may omit these fields; consumers must preserve the existing behavior when they are absent or invalid.
`branch` selects the base branch for the first run.
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index bad6ccdbeeca..160d9c47c668 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -12015,7 +12015,7 @@ snapshots:
scenes-app-inbox-detail--run-failed--dark:
hash: v1.k794b7964.e82d63bd21d2d441d8c2c13cc22279805acdec1bb01ec04f2fbef3ad418adaeb.hXPwtY6-D5RdgQZYCCVXmnz_VtcZdDkAGuQlkAizdvY
scenes-app-inbox-detail--run-failed--light:
- hash: v1.k794b7964.ca50dccb1cd03f484f30cf956203d8db51774dc6cfdb3ee02f3309053d23ac14.SZtbTt4qO-2rSWV4i_0bt7-DkKS7kx9tQgYFzR6O1OQ
+ hash: v1.k794b7964.b11557ace2e0d1b029cd6bd3cf845b6a7461ac0e47a68a29e6bf1590e9ab0609.X4592bipRdc45hq0nADPQTsXiYjWDKLYtwWnhSSkSMA
scenes-app-inbox-detail--run-in-progress--dark:
hash: v1.k794b7964.e00dcc751d8cf2623a6418e035c4004c6a094603ecad497dcb8d9d0fb3bdce49.xQef0B2Lc41HTu1Gn_Ii1j2oyDz28Ku4XYeEgKgqHa8
scenes-app-inbox-detail--run-in-progress--light:
diff --git a/posthog/temporal/tests/ai/test_module_integrity.py b/posthog/temporal/tests/ai/test_module_integrity.py
index e01eb58f0b11..4886c6524c7d 100644
--- a/posthog/temporal/tests/ai/test_module_integrity.py
+++ b/posthog/temporal/tests/ai/test_module_integrity.py
@@ -154,6 +154,7 @@ class TestSignalsProductModuleIntegrity:
"get_embedding_activity",
"match_signal_to_report_activity",
"mark_report_failed_activity",
+ "notify_research_startup_failure_activity",
"read_signals_from_s3_activity",
"check_report_quota_gate_activity",
"check_report_task_settlement_activity",
diff --git a/products/signals/backend/migrations/0211_agent_startup_failure.py b/products/signals/backend/migrations/0211_agent_startup_failure.py
new file mode 100644
index 000000000000..e108eab9d98e
--- /dev/null
+++ b/products/signals/backend/migrations/0211_agent_startup_failure.py
@@ -0,0 +1,13 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [("signals", "0210_monitoring_scout_recovery_index")]
+
+ operations = [
+ migrations.AddField(
+ model_name="signalreport",
+ name="startup_failure",
+ field=models.JSONField(blank=True, null=True),
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index 65999d5fd901..54d6d1970be7 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0210_monitoring_scout_recovery_index
+0211_agent_startup_failure
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index 26622040292d..990b7f6bc376 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -423,6 +423,7 @@ class SignalReport(UUIDModel):
title = models.TextField(null=True, blank=True)
summary = models.TextField(null=True, blank=True)
error = models.TextField(null=True, blank=True)
+ startup_failure = models.JSONField(null=True, blank=True)
# The charts this report currently shows, each a `ReportChart` (see report_charts.py). Part of
# the report's content rather than its artefact log: a chart illustrates the summary, so it is
# replaced with the summary rather than accumulating versions beside it. `summary` places one
@@ -618,6 +619,8 @@ class SignalReport(UUIDModel):
self.last_run_at = timezone.now()
self.signals_at_run = self.signal_count + signals_at_run_increment
self.run_count += 1
+ self.startup_failure = None
+ updated_fields.add("startup_failure")
updated_fields.update(["last_run_at", "signals_at_run", "run_count"])
# A summary run paused mid-workflow by the self-driving credits quota gate returns to
@@ -634,7 +637,8 @@ class SignalReport(UUIDModel):
self.title = title
self.summary = summary
self.error = None
- updated_fields.update(["title", "summary", "error"])
+ self.startup_failure = None
+ updated_fields.update(["title", "summary", "error", "startup_failure"])
# A `None` title or summary keeps the current one, so a run that did no research (e.g. no
# repository selected) does not erase the content the report is searched and deduplicated by.
diff --git a/products/signals/backend/report_generation/AGENTS.md b/products/signals/backend/report_generation/AGENTS.md
index a62fcd90a9ef..ea01c4d05619 100644
--- a/products/signals/backend/report_generation/AGENTS.md
+++ b/products/signals/backend/report_generation/AGENTS.md
@@ -31,6 +31,13 @@ It is exercised locally via management commands, and it is also used by the prod
The repository used for research is tracked separately via the `repo_selection` artefact.
+ Startup failures cross the Tasks facade as `AgentStartupFailed`. The caller activity preserves
+ their typed details in `ResearchAgentStartupFailed`; transient sandbox failures retain the extended
+ `ResearchSandboxUnavailable` retry budget. Only failures classified as `requires_action` skip activity retries; a new run can succeed after the configuration or credentials are corrected.
+ The summary workflow persists the terminal failure, then retries its deduplicated notice in a separate
+ activity through `backend/startup_failure.py`. Recovery requires worker-attested startup success.
+ Keep this lifecycle in the caller, outside the research prompt flow.
+
- `ownership_reviewers.py`
Matches a finding's relevant code paths against the repository's `owners.yaml` and CODEOWNERS on the connected GitHub repository. When both name different project members, it suggests the `owners.yaml` owner first and the CODEOWNERS owner second. The artefact write path preserves human pins, exclusions, and confirmed lists during later research.
diff --git a/products/signals/backend/serializers.py b/products/signals/backend/serializers.py
index 69e55a1de05f..dfe11ebf1302 100644
--- a/products/signals/backend/serializers.py
+++ b/products/signals/backend/serializers.py
@@ -1465,7 +1465,45 @@ class SignalReportSerializerExtension(OpenApiSerializerExtension):
return schema
+class StartupProvider(TextChoices):
+ CLAUDE = "claude", "Claude"
+ CODEX = "codex", "Codex"
+ PI = "pi", "Pi"
+
+
+class StartupPhase(TextChoices):
+ SESSION_SETUP = "session_setup", "Session setup"
+ SDK_INITIALIZATION = "sdk_initialization", "SDK initialization"
+ SETUP_HOOKS = "setup_hooks", "Setup hooks"
+
+
+class StartupRetryability(TextChoices):
+ REQUIRES_ACTION = "requires_action", "Requires action"
+ TRANSIENT = "transient", "Transient"
+ UNKNOWN = "unknown", "Unknown"
+
+
+class ReportStartupFailureSerializer(serializers.Serializer):
+ provider = serializers.ChoiceField(
+ choices=StartupProvider.choices, help_text="Agent provider that failed to start."
+ )
+ phase = serializers.ChoiceField(choices=StartupPhase.choices, help_text="Startup stage that failed.")
+ reason = serializers.CharField(allow_null=True, help_text="Validated provider reason code, when available.")
+ retryability = serializers.ChoiceField(
+ choices=StartupRetryability.choices, help_text="Whether an unchanged startup can be retried."
+ )
+ task_id = serializers.UUIDField(help_text="Research task that could not start.")
+ run_id = serializers.UUIDField(help_text="Failed research task run.")
+ message = serializers.CharField(help_text="Safe explanation and suggested next action.")
+
+
class SignalReportSerializer(serializers.ModelSerializer):
+ startup_failure = ReportStartupFailureSerializer(
+ read_only=True,
+ allow_null=True,
+ required=False,
+ help_text="Startup failure from the latest failed research attempt, when available.",
+ )
reviewer_assignment = serializers.SerializerMethodField(
help_text="Reviewer control and state token for preparing an automatic proposal."
)
@@ -1679,6 +1717,7 @@ class SignalReportSerializer(serializers.ModelSerializer):
"signal_count", # Used for occurrence count
"signals_at_run", # Snooze threshold: re-promote when signal_count >= this value
"collapsed_note_count", # Scout notes the work log dropped, rendered as one line instead
+ "startup_failure",
"created_at",
"updated_at",
"artefact_count",
diff --git a/products/signals/backend/startup_failure.py b/products/signals/backend/startup_failure.py
new file mode 100644
index 000000000000..0936f1186082
--- /dev/null
+++ b/products/signals/backend/startup_failure.py
@@ -0,0 +1,210 @@
+import hashlib
+from collections.abc import Mapping
+from dataclasses import asdict
+from datetime import datetime
+from uuid import UUID
+
+from django.db import connection, transaction
+from django.utils import timezone
+from django.utils.dateparse import parse_datetime
+
+from pydantic import TypeAdapter, ValidationError
+from pydantic.dataclasses import dataclass
+from temporalio.exceptions import ActivityError, ApplicationError
+
+from posthog.models import HealthIssue, OrganizationMembership, Team
+
+from products.notifications.backend.facade.api import (
+ NotificationData,
+ NotificationType,
+ RecipientsResolver,
+ TargetType,
+ create_notification,
+)
+from products.tasks.backend.facade import api as tasks_facade
+from products.tasks.backend.facade.contracts import AgentStartupFailure
+
+RESEARCH_STARTUP_FAILURE = "ResearchAgentStartupFailed"
+RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE = "ResearchSandboxUnavailable"
+ISSUE_KIND = "signals_agent_startup_failed"
+SCOPE_KEYS = ["repository", "provider", "runtime"]
+
+
+@dataclass(frozen=True, kw_only=True)
+class ResearchStartupFailure:
+ failure: AgentStartupFailure
+ task_id: UUID
+ run_id: UUID
+ repository: str
+
+
+_failure_adapter = TypeAdapter(ResearchStartupFailure)
+
+
+def research_startup_failure(error: Exception) -> ResearchStartupFailure | None:
+ cause = error.cause if isinstance(error, ActivityError) else error
+ if (
+ not isinstance(cause, ApplicationError)
+ or cause.type not in (RESEARCH_STARTUP_FAILURE, RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE)
+ or not cause.details
+ ):
+ return None
+ try:
+ return _failure_adapter.validate_python(cause.details[0])
+ except ValidationError:
+ return None
+
+
+def startup_failure_message(failure: AgentStartupFailure) -> str:
+ if failure.reason == "credential_unavailable":
+ return "Research could not start because the agent could not obtain its credentials. Follow the provider's reconnect or token-relay instructions in the linked run, then retry."
+ if failure.reason == "credential_service_unavailable":
+ return "Research could not start because the provider's credential service was temporarily unavailable. Retry from the linked run."
+ explanations = {
+ "proxy_invalid": "The agent's proxy configuration is invalid.",
+ "managed_settings_invalid": "The agent's managed settings are invalid.",
+ "shell_tool_missing": "A shell tool required by the agent is unavailable.",
+ "gateway_signin_required": "The agent needs to sign in to its provider.",
+ "gateway_access_denied": "The provider denied access while loading the agent's managed settings.",
+ "org_pin_api_key_conflict": "The agent's API key conflicts with its organization settings.",
+ "org_pin_mismatch": "The agent's organization settings do not match its credentials.",
+ "cli_version_too_old": "The agent requires a newer provider runtime.",
+ "bypass_root": "The provider rejected the agent's permission configuration.",
+ }
+ explanation = explanations.get(failure.reason or "") if failure.provider == "claude" else None
+ if explanation and failure.retryability == "requires_action":
+ return f"Research could not start. {explanation} Check the linked run and update the configuration before retrying."
+ return "Research stopped before the agent could start. The provider did not report a confirmed cause. Check the linked run for details before retrying."
+
+
+def report_startup_failure(failure: ResearchStartupFailure) -> dict[str, object]:
+ return {
+ **asdict(failure.failure),
+ "task_id": str(failure.task_id),
+ "run_id": str(failure.run_id),
+ "message": startup_failure_message(failure.failure),
+ }
+
+
+class ProjectOrganizationAdminRecipients(RecipientsResolver):
+ def resolve(self, target_type: TargetType, target_id: str, team_id: int | None) -> list[int]:
+ team = Team.objects.get(id=team_id)
+ admins = OrganizationMembership.objects.filter(
+ organization_id=team.organization_id, level__gte=OrganizationMembership.Level.ADMIN
+ ).values_list("user_id", flat=True)
+ return list(team.all_users_with_access().filter(id__in=admins).values_list("id", flat=True))
+
+
+def _lock_scope(team_id: int, scope_hash: str) -> None:
+ key = int.from_bytes(hashlib.sha256(f"{ISSUE_KIND}:{team_id}:{scope_hash}".encode()).digest()[:8], signed=True)
+ with connection.cursor() as cursor:
+ cursor.execute("SELECT pg_advisory_xact_lock(%s)", [key])
+
+
+def _started_at(
+ state: Mapping[str, object], fallback: datetime | None = None, *, key: str = "startup_started_at"
+) -> datetime | None:
+ value = state.get(key)
+ try:
+ parsed = parse_datetime(value) if isinstance(value, str) else None
+ except ValueError:
+ parsed = None
+ return parsed if parsed is not None and timezone.is_aware(parsed) else fallback
+
+
+def notify_research_startup_failure(team_id: int, report_id: str, failure: ResearchStartupFailure) -> None:
+ run = tasks_facade.get_task_run(failure.run_id, team_id=team_id)
+ if run is None or run.task_id != failure.task_id:
+ return
+ attempt = run.state.get("agent_startup_attempt")
+ started_at = (
+ _started_at(attempt, run.created_at, key="started_at")
+ if isinstance(attempt, dict)
+ else _started_at(run.state, run.created_at)
+ )
+ if started_at is None:
+ return
+ runtime = (
+ attempt.get("runtime", "unknown") if isinstance(attempt, dict) else run.state.get("startup_runtime", "unknown")
+ )
+ if runtime not in ("gvisor", "vm"):
+ runtime = "unknown"
+ scope = {"repository": failure.repository.lower(), "provider": failure.failure.provider, "runtime": runtime}
+ scope_hash = HealthIssue.compute_unique_hash(ISSUE_KIND, scope, SCOPE_KEYS)
+ issue_payload = {**scope, "category": failure.failure.reason or failure.failure.phase}
+ issue_hash = HealthIssue.compute_unique_hash(ISSUE_KIND, issue_payload, [*SCOPE_KEYS, "category"])
+ with transaction.atomic():
+ _lock_scope(team_id, scope_hash)
+ successful_at = tasks_facade.latest_agent_startup_success(
+ team_id=team_id, repository=scope["repository"], provider=scope["provider"], runtime=scope["runtime"]
+ )
+ if successful_at is not None:
+ recovered = HealthIssue.objects.filter(
+ team_id=team_id,
+ kind=ISSUE_KIND,
+ status=HealthIssue.Status.ACTIVE,
+ payload__repository=scope["repository"],
+ payload__provider=scope["provider"],
+ payload__runtime=scope["runtime"],
+ )
+ for recovered_issue in recovered:
+ previous_started_at = _started_at(recovered_issue.payload)
+ if previous_started_at is not None and previous_started_at <= successful_at:
+ recovered_issue.status = HealthIssue.Status.RESOLVED
+ recovered_issue.resolved_at = timezone.now()
+ recovered_issue.save(update_fields=["status", "resolved_at", "updated_at"])
+ if successful_at is not None and successful_at >= started_at:
+ return
+ issue = HealthIssue.objects.filter(
+ team_id=team_id, kind=ISSUE_KIND, unique_hash=issue_hash, status=HealthIssue.Status.ACTIVE
+ ).first()
+ if issue is None:
+ issue = HealthIssue.objects.create(
+ team_id=team_id,
+ kind=ISSUE_KIND,
+ unique_hash=issue_hash,
+ severity=HealthIssue.Severity.WARNING,
+ payload={**issue_payload, "startup_started_at": started_at.isoformat(), "report_id": report_id},
+ )
+ elif started_at > (_started_at(issue.payload) or issue.created_at):
+ issue.payload.update(startup_started_at=started_at.isoformat(), report_id=report_id)
+ issue.save(update_fields=["payload", "updated_at"])
+ create_notification(
+ NotificationData(
+ team_id=team_id,
+ notification_type=NotificationType.PIPELINE_FAILURE,
+ title="Research agent could not start",
+ body=startup_failure_message(failure.failure),
+ target_type=TargetType.TEAM,
+ target_id=str(team_id),
+ source_url=f"/project/{team_id}/inbox/{report_id}",
+ idempotency_key=f"{ISSUE_KIND}:{issue.id}",
+ resolver=ProjectOrganizationAdminRecipients(),
+ )
+ )
+
+
+def reconcile_research_startups(team_id: int, repository: str) -> None:
+ issues = HealthIssue.objects.filter(
+ team_id=team_id,
+ kind=ISSUE_KIND,
+ status=HealthIssue.Status.ACTIVE,
+ payload__repository=repository.lower(),
+ )
+ for issue in issues:
+ with transaction.atomic():
+ _lock_scope(team_id, HealthIssue.compute_unique_hash(ISSUE_KIND, issue.payload, SCOPE_KEYS))
+ issue.refresh_from_db()
+ started_at = _started_at(issue.payload)
+ if issue.status != HealthIssue.Status.ACTIVE or started_at is None:
+ continue
+ successful_at = tasks_facade.latest_agent_startup_success(
+ team_id=team_id,
+ repository=repository.lower(),
+ provider=issue.payload["provider"],
+ runtime=issue.payload["runtime"],
+ )
+ if successful_at is not None and successful_at >= started_at:
+ issue.status = HealthIssue.Status.RESOLVED
+ issue.resolved_at = timezone.now()
+ issue.save(update_fields=["status", "resolved_at", "updated_at"])
diff --git a/products/signals/backend/temporal/__init__.py b/products/signals/backend/temporal/__init__.py
index 0da729d85c57..5591f606e5c8 100644
--- a/products/signals/backend/temporal/__init__.py
+++ b/products/signals/backend/temporal/__init__.py
@@ -125,6 +125,7 @@ from products.signals.backend.temporal.summary import (
mark_report_pending_input_activity,
mark_report_ready_activity,
maybe_autostart_implementation_activity,
+ notify_research_startup_failure_activity,
publish_report_completed_activity,
publish_report_spend_activity,
report_has_assigned_signals_activity,
@@ -196,6 +197,7 @@ ACTIVITIES = [
get_embedding_activity,
match_signal_to_report_activity,
mark_report_failed_activity,
+ notify_research_startup_failure_activity,
read_signals_from_s3_activity,
check_report_quota_gate_activity,
check_report_task_settlement_activity,
diff --git a/products/signals/backend/temporal/agentic/report.py b/products/signals/backend/temporal/agentic/report.py
index 01026ef05f0e..351411452d30 100644
--- a/products/signals/backend/temporal/agentic/report.py
+++ b/products/signals/backend/temporal/agentic/report.py
@@ -1,5 +1,5 @@
import json
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
from datetime import datetime
from typing import Any, TypeVar
from uuid import UUID
@@ -73,6 +73,12 @@ from products.signals.backend.report_steering import ReportSteering, load_resear
from products.signals.backend.report_tags import team_tag_vocabulary
from products.signals.backend.reviewer_rules import ReviewerProposal, ReviewerRules, StaleReviewerProposal
from products.signals.backend.source_suggestions import unused_suggestable_products
+from products.signals.backend.startup_failure import (
+ RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE as RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE,
+ RESEARCH_STARTUP_FAILURE,
+ reconcile_research_startups,
+ startup_failure_message,
+)
from products.signals.backend.supersession import research_implementation_context
from products.signals.backend.temporal.agentic import (
SIGNALS_REPORT_RESEARCH_ENV_NAME,
@@ -86,7 +92,12 @@ from products.signals.backend.typed_report_links import (
outgoing_links,
)
from products.tasks.backend.facade import api as tasks_facade
-from products.tasks.backend.facade.agents import CustomPromptSandboxContext, SandboxRunFailed, turn_failure_error_type
+from products.tasks.backend.facade.agents import (
+ AgentStartupFailed,
+ CustomPromptSandboxContext,
+ SandboxRunFailed,
+ turn_failure_error_type,
+)
logger = structlog.get_logger(__name__)
@@ -1115,9 +1126,6 @@ def _capture_research_steering_attached(*, team_id: int, report_id: str, steerin
logger.exception("Failed to capture signals_research_steering_attached", report_id=report_id)
-RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE = "ResearchSandboxUnavailable"
-
-
@temporalio.activity.defn
@scoped_temporal()
@close_db_connections
@@ -1300,6 +1308,18 @@ async def run_agentic_report_activity(input: RunAgenticReportInput) -> RunAgenti
charts_enabled=True,
tags=result.tags,
)
+ except AgentStartupFailed as error:
+ raise ApplicationError(
+ startup_failure_message(error.failure),
+ {
+ "failure": asdict(error.failure),
+ "task_id": error.task_id,
+ "run_id": error.run_id,
+ "repository": input.repo_selection.repository,
+ },
+ type=RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE if error.transient else RESEARCH_STARTUP_FAILURE,
+ non_retryable=error.failure.retryability == "requires_action",
+ ) from error
except SandboxRunFailed as error:
logger.exception(
"signals agentic report failed",
@@ -1324,3 +1344,11 @@ async def run_agentic_report_activity(input: RunAgenticReportInput) -> RunAgenti
if error_type is not None:
raise ApplicationError(str(error), type=error_type) from error
raise
+ finally:
+ if input.repo_selection.repository:
+ try:
+ await database_sync_to_async(reconcile_research_startups, thread_sensitive=False)(
+ input.team_id, input.repo_selection.repository
+ )
+ except Exception:
+ logger.warning("signals_startup_recovery_failed", team_id=input.team_id, exc_info=True)
diff --git a/products/signals/backend/temporal/summary.py b/products/signals/backend/temporal/summary.py
index ff9b6dd33416..db5022a5b26a 100644
--- a/products/signals/backend/temporal/summary.py
+++ b/products/signals/backend/temporal/summary.py
@@ -68,6 +68,18 @@ from products.signals.backend.spend import (
submit_signal_metadata_updates,
)
from products.signals.backend.stack_plan import create_layer_reports, start_unblocked_layers_of_plan
+from products.signals.backend.startup_failure import (
+ ResearchStartupFailure,
+ notify_research_startup_failure,
+ report_startup_failure,
+ research_startup_failure,
+ startup_failure_message,
+)
+from products.signals.backend.task_run_artefacts import (
+ SIGNALS_PRODUCT,
+ TASK_RUN_TYPE_RESEARCH,
+ append_task_run_artefact,
+)
from products.signals.backend.temporal import metrics
from products.signals.backend.temporal.agentic.report import (
RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE,
@@ -835,6 +847,7 @@ class SignalReportSummaryWorkflow:
)
return has_new_signals
except Exception as e:
+ startup_failure = research_startup_failure(e)
await workflow.execute_activity(
mark_report_failed_activity,
MarkReportFailedInput(
@@ -847,12 +860,28 @@ class SignalReportSummaryWorkflow:
else "agentic_activity_error"
),
error_type=_activity_error_type(e),
+ startup_failure=startup_failure,
signal_count=signal_count,
source_products=source_products,
),
start_to_close_timeout=timedelta(minutes=1),
retry_policy=RetryPolicy(maximum_attempts=3),
)
+ if startup_failure:
+ try:
+ await workflow.execute_activity(
+ notify_research_startup_failure_activity,
+ NotifyResearchStartupFailureInput(
+ team_id=inputs.team_id, report_id=inputs.report_id, failure=startup_failure
+ ),
+ start_to_close_timeout=timedelta(minutes=1),
+ schedule_to_close_timeout=timedelta(minutes=10),
+ retry_policy=RetryPolicy(
+ initial_interval=timedelta(seconds=5), maximum_interval=timedelta(minutes=1)
+ ),
+ )
+ except ActivityError:
+ workflow.logger.exception("Failed to deliver research startup notification")
raise
@@ -1353,6 +1382,23 @@ class MarkReportFailedInput:
error_type: str | None = None
signal_count: int = 0
source_products: list[str] = field(default_factory=list)
+ startup_failure: ResearchStartupFailure | None = None
+
+
+@frozen
+class NotifyResearchStartupFailureInput:
+ team_id: int
+ report_id: str
+ failure: ResearchStartupFailure
+
+
+@scoped_temporal()
+@close_db_connections
+async def notify_research_startup_failure_activity(input: NotifyResearchStartupFailureInput) -> None:
+ await database_sync_to_async(notify_research_startup_failure, thread_sensitive=False)(
+ input.team_id, input.report_id, input.failure
+ )
@temporalio.activity.defn
@@ -1367,8 +1413,22 @@ async def mark_report_failed_activity(input: MarkReportFailedInput) -> None:
report = SignalReport.objects.select_for_update().get(id=input.report_id, team_id=input.team_id)
if report.status == SignalReport.Status.FAILED:
return report.run_count, True
- updated_fields = report.transition_to(SignalReport.Status.FAILED, error=input.error)
+ updated_fields = report.transition_to(
+ SignalReport.Status.FAILED,
+ error=startup_failure_message(input.startup_failure.failure) if input.startup_failure else input.error,
+ )
+ report.startup_failure = report_startup_failure(input.startup_failure) if input.startup_failure else None
+ updated_fields.append("startup_failure")
report.save(update_fields=updated_fields)
+ if input.startup_failure:
+ append_task_run_artefact(
+ team_id=input.team_id,
+ report_id=input.report_id,
+ product=SIGNALS_PRODUCT,
+ type=TASK_RUN_TYPE_RESEARCH,
+ task_id=str(input.startup_failure.task_id),
+ run_id=str(input.startup_failure.run_id),
+ )
return report.run_count, False
run_count, was_already_failed = await database_sync_to_async(do_update, thread_sensitive=False)()
diff --git a/products/signals/backend/test/test_agentic_report_activity.py b/products/signals/backend/test/test_agentic_report_activity.py
index baa19d99e3e2..06eea191e72a 100644
--- a/products/signals/backend/test/test_agentic_report_activity.py
+++ b/products/signals/backend/test/test_agentic_report_activity.py
@@ -61,6 +61,7 @@ from products.signals.backend.report_generation.research import (
)
from products.signals.backend.report_generation.select_repo import RepoSelectionResult
from products.signals.backend.report_metrics import REPORT_METRIC_GOAL_FIELDS, ReportMetric
+from products.signals.backend.startup_failure import RESEARCH_STARTUP_FAILURE, research_startup_failure
from products.signals.backend.supersession import ImplementationResearchContext
from products.signals.backend.temporal.agentic.report import (
RESEARCH_MCP_SCOPES,
@@ -86,7 +87,8 @@ from products.signals.backend.temporal.summary import (
mark_report_ready_activity,
)
from products.signals.backend.temporal.types import SignalData
-from products.tasks.backend.facade.agents import CustomPromptSandboxContext, SandboxRunFailed
+from products.tasks.backend.facade.agents import AgentStartupFailed, CustomPromptSandboxContext, SandboxRunFailed
+from products.tasks.backend.facade.contracts import AgentStartupFailure
from products.tasks.backend.models import Task # tach-ignore
@@ -749,7 +751,14 @@ async def test_select_repository_activity_does_not_raise_with_only_user_integrat
@pytest.mark.django_db
@pytest.mark.parametrize(
"sandbox_error_type,expected_transient",
- [("RepositoryCloneError", True), ("RepositoryAccessError", False)],
+ [
+ ("RepositoryCloneError", True),
+ ("RepositoryAccessError", False),
+ ("startup_requires_action", False),
+ ("startup_unknown", False),
+ ("startup_transient", True),
+ ("startup_unknown_transient_sandbox", True),
+ ],
)
async def test_run_agentic_report_activity_flags_transient_sandbox_failures(
monkeypatch, ateam, sandbox_error_type, expected_transient
@@ -763,6 +772,25 @@ async def test_run_agentic_report_activity_flags_transient_sandbox_failures(
monkeypatch.setattr("products.signals.backend.temporal.agentic.report.resolve_user_id_for_team", lambda team_id: 1)
async def fake_run_multi_turn_research(*args, **kwargs):
+ if sandbox_error_type.startswith("startup_"):
+ requires_action = sandbox_error_type == "startup_requires_action"
+ raise AgentStartupFailed(
+ failure=AgentStartupFailure(
+ provider="claude",
+ phase="sdk_initialization",
+ reason="proxy_invalid" if requires_action else None,
+ retryability="requires_action"
+ if requires_action
+ else "transient"
+ if sandbox_error_type == "startup_transient"
+ else "unknown",
+ ),
+ task_id=str(uuid4()),
+ run_id=str(uuid4()),
+ error_type="SandboxExecutionError"
+ if sandbox_error_type == "startup_unknown_transient_sandbox"
+ else None,
+ )
raise SandboxRunFailed("TaskRun reached terminal status=failed", error_type=sandbox_error_type)
monkeypatch.setattr(
@@ -780,7 +808,14 @@ async def test_run_agentic_report_activity_flags_transient_sandbox_failures(
)
)
- if expected_transient:
+ if sandbox_error_type.startswith("startup_"):
+ assert isinstance(error.value, ApplicationError)
+ assert error.value.type == (
+ RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE if expected_transient else RESEARCH_STARTUP_FAILURE
+ )
+ assert error.value.non_retryable is (sandbox_error_type == "startup_requires_action")
+ assert research_startup_failure(error.value) is not None
+ elif expected_transient:
assert isinstance(error.value, ApplicationError)
assert error.value.type == RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE
else:
diff --git a/products/signals/backend/test/test_report_telemetry.py b/products/signals/backend/test/test_report_telemetry.py
index ddade984b5d4..3ee2dfd12274 100644
--- a/products/signals/backend/test/test_report_telemetry.py
+++ b/products/signals/backend/test/test_report_telemetry.py
@@ -8,19 +8,24 @@ from posthog.sync import database_sync_to_async
from products.signals.backend.models import SignalReport, SignalReportArtefact, SignalReportCheck, SignalScoutConfig
from products.signals.backend.report_check_agent import FALLBACK_CHECK_SKILL_NAME
+from products.signals.backend.startup_failure import ResearchStartupFailure
from products.signals.backend.temporal.summary import (
MarkReportFailedInput,
MarkReportInProgressInput,
MarkReportPendingInput,
MarkReportReadyInput,
+ NotifyResearchStartupFailureInput,
ResetReportToPotentialInput,
mark_report_failed_activity,
mark_report_in_progress_activity,
mark_report_pending_input_activity,
mark_report_ready_activity,
+ notify_research_startup_failure_activity,
reset_report_to_potential_activity,
)
from products.skills.backend.models.skills import LLMSkill
+from products.tasks.backend.facade.contracts import AgentStartupFailure
+from products.tasks.backend.models import Task, TaskRun
PIPELINE_MODULE_PATH = "products.signals.backend.temporal.summary"
@@ -31,6 +36,55 @@ CHART_PAYLOAD = {
}
+async def test_startup_failure_is_durable_when_notification_delivery_fails(ateam):
+ task = await Task.objects.acreate(team=ateam, title="Example research", repository="example/research")
+ run = await TaskRun.objects.acreate(team=ateam, task=task, status=TaskRun.Status.FAILED)
+ report = await database_sync_to_async(SignalReport.objects.create)(
+ team=ateam,
+ status=SignalReport.Status.IN_PROGRESS,
+ signal_count=1,
+ total_weight=1,
+ )
+ failure = ResearchStartupFailure(
+ failure=AgentStartupFailure(provider="claude", phase="sdk_initialization", reason=None, retryability="unknown"),
+ task_id=task.id,
+ run_id=run.id,
+ repository="example/research",
+ )
+ with (
+ patch(
+ f"{PIPELINE_MODULE_PATH}.notify_research_startup_failure", side_effect=RuntimeError("delivery unavailable")
+ ),
+ patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture"),
+ ):
+ for _ in range(2):
+ await mark_report_failed_activity(
+ MarkReportFailedInput(
+ team_id=ateam.id,
+ report_id=str(report.id),
+ error="SECRET raw provider output",
+ startup_failure=failure,
+ )
+ )
+ with pytest.raises(RuntimeError, match="delivery unavailable"):
+ await notify_research_startup_failure_activity(
+ NotifyResearchStartupFailureInput(team_id=ateam.id, report_id=str(report.id), failure=failure)
+ )
+ await report.arefresh_from_db()
+ assert report.status == SignalReport.Status.FAILED
+ assert report.startup_failure is not None
+ assert report.error is not None
+ assert report.startup_failure["run_id"] == str(failure.run_id)
+ assert report.startup_failure["retryability"] == "unknown"
+ assert report.error == report.startup_failure["message"]
+ assert "SECRET" not in report.error
+ artefact = await SignalReportArtefact.objects.aget(report=report, type="task_run")
+ assert artefact.task_id == task.id
+ assert str(run.id) in artefact.content
+
+
@pytest.mark.asyncio
@pytest.mark.django_db
@pytest.mark.parametrize(
diff --git a/products/signals/backend/test/test_startup_failure.py b/products/signals/backend/test/test_startup_failure.py
new file mode 100644
index 000000000000..60581023befd
--- /dev/null
+++ b/products/signals/backend/test/test_startup_failure.py
@@ -0,0 +1,247 @@
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import replace
+from datetime import timedelta
+from threading import Barrier
+from uuid import uuid4
+
+import pytest
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from django.db import close_old_connections
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from posthog.models import HealthIssue, Organization, OrganizationMembership, Team, User
+
+from products.notifications.backend.facade.api import create_notification
+from products.notifications.backend.facade.testing import stored_notifications_for_team
+from products.signals.backend.startup_failure import (
+ ISSUE_KIND,
+ ResearchStartupFailure,
+ notify_research_startup_failure,
+ reconcile_research_startups,
+)
+from products.tasks.backend.facade.contracts import AgentStartupFailure
+from products.tasks.backend.models import Task, TaskRun
+
+
+class TestResearchStartupFailure(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.flag = patch("posthoganalytics.feature_enabled", return_value=True)
+ self.flag.start()
+ self.addCleanup(self.flag.stop)
+ OrganizationMembership.objects.filter(organization=self.organization, user=self.user).update(
+ level=OrganizationMembership.Level.ADMIN
+ )
+ self.task = Task.objects.create(
+ team=self.team, title="Example research", repository="example/research", created_by=self.user
+ )
+ self.started_at = timezone.now()
+ self.task_run = TaskRun.objects.create(
+ team=self.team,
+ task=self.task,
+ status=TaskRun.Status.FAILED,
+ state={"startup_started_at": self.started_at.isoformat(), "startup_runtime": "gvisor"},
+ )
+ self.failure = ResearchStartupFailure(
+ failure=AgentStartupFailure(
+ provider="claude", phase="sdk_initialization", reason="proxy_invalid", retryability="requires_action"
+ ),
+ task_id=self.task.id,
+ run_id=self.task_run.id,
+ repository="example/research",
+ )
+ self.report_id = str(uuid4())
+
+ def notify(self) -> None:
+ notify_research_startup_failure(self.team.id, self.report_id, self.failure)
+
+ def test_repeated_failures_notify_only_admins_once_per_episode(self) -> None:
+ member = User.objects.create(email="[email protected]")
+ OrganizationMembership.objects.create(
+ organization=self.organization, user=member, level=OrganizationMembership.Level.MEMBER
+ )
+ with patch(
+ "products.signals.backend.startup_failure.create_notification", wraps=create_notification
+ ) as dispatch:
+ self.notify()
+ self.notify()
+ assert dispatch.call_args.args[0].priority == "normal"
+
+ issue = HealthIssue.objects.get(team=self.team, kind=ISSUE_KIND)
+ (event,) = stored_notifications_for_team(self.team.id)
+ assert issue.status == HealthIssue.Status.ACTIVE
+ assert event.resolved_user_ids == [self.user.id]
+ assert event.source_url == f"/project/{self.team.id}/inbox/{self.report_id}"
+ assert "proxy configuration" in event.body
+
+ @parameterized.expand(
+ [
+ ("older", "claude", "gvisor", -1, False),
+ ("other_provider", "codex", "gvisor", 1, False),
+ ("other_runtime", "claude", "vm", 1, False),
+ ("matching", "claude", "gvisor", 1, True),
+ ]
+ )
+ def test_only_newer_matching_startup_resolves(
+ self, name: str, provider: str, runtime: str, offset: int, resolves: bool
+ ) -> None:
+ self.notify()
+ TaskRun.objects.create(
+ team=self.team,
+ task=self.task,
+ state={
+ "agent_startup_success": {
+ "started_at": (self.started_at + timedelta(seconds=offset)).isoformat(),
+ "completed_at": (self.started_at + timedelta(seconds=offset + 1)).isoformat(),
+ "repository": "example/research",
+ "provider": provider,
+ "runtime": runtime,
+ }
+ },
+ )
+ reconcile_research_startups(self.team.id, "example/research")
+ issue = HealthIssue.objects.get(team=self.team, kind=ISSUE_KIND)
+ assert (issue.status == HealthIssue.Status.RESOLVED) is resolves
+ self.notify()
+ assert len(stored_notifications_for_team(self.team.id)) == 1
+
+ if resolves:
+ self.task_run.state["startup_started_at"] = (self.started_at + timedelta(seconds=3)).isoformat()
+ self.task_run.save(update_fields=["state"])
+ self.notify()
+ assert HealthIssue.objects.filter(team=self.team, status=HealthIssue.Status.ACTIVE).count() == 1
+ assert len(stored_notifications_for_team(self.team.id)) == 2
+
+ def test_success_observed_before_delayed_failure_suppresses_notice(self) -> None:
+ TaskRun.objects.create(
+ team=self.team,
+ task=self.task,
+ state={
+ "agent_startup_success": {
+ "started_at": (self.started_at + timedelta(seconds=1)).isoformat(),
+ "completed_at": (self.started_at + timedelta(seconds=2)).isoformat(),
+ "repository": "example/research",
+ "provider": "claude",
+ "runtime": "gvisor",
+ }
+ },
+ )
+ self.notify()
+ assert not HealthIssue.objects.filter(team=self.team, kind=ISSUE_KIND).exists()
+ assert not stored_notifications_for_team(self.team.id)
+
+ def test_delivery_failure_retains_issue_and_can_be_retried(self) -> None:
+ with patch(
+ "products.signals.backend.startup_failure.create_notification", side_effect=RuntimeError("unavailable")
+ ):
+ with pytest.raises(RuntimeError, match="unavailable"):
+ self.notify()
+ assert HealthIssue.objects.filter(team=self.team, status=HealthIssue.Status.ACTIVE).count() == 1
+ assert not stored_notifications_for_team(self.team.id)
+ self.notify()
+ assert len(stored_notifications_for_team(self.team.id)) == 1
+
+ def test_recovery_between_failures_starts_a_new_episode_without_reconciliation(self) -> None:
+ self.notify()
+ previous = HealthIssue.objects.get(team=self.team, kind=ISSUE_KIND)
+ self.task_run.state.update(
+ agent_startup_success={
+ "repository": self.task.repository,
+ "provider": "claude",
+ "runtime": "gvisor",
+ "started_at": (self.started_at + timedelta(seconds=1)).isoformat(),
+ "completed_at": (self.started_at + timedelta(seconds=2)).isoformat(),
+ },
+ agent_startup_attempt={
+ "repository": self.task.repository,
+ "provider": "claude",
+ "runtime": "gvisor",
+ "started_at": (self.started_at + timedelta(seconds=3)).isoformat(),
+ },
+ )
+ self.task_run.save(update_fields=["state"])
+ self.notify()
+ previous.refresh_from_db()
+ assert previous.status == HealthIssue.Status.RESOLVED
+ assert (
+ HealthIssue.objects.filter(team=self.team, kind=ISSUE_KIND, status=HealthIssue.Status.ACTIVE).count() == 1
+ )
+ assert len(stored_notifications_for_team(self.team.id)) == 2
+
+ def test_agent_reported_success_is_not_recovery_evidence(self) -> None:
+ self.notify()
+ TaskRun.objects.create(
+ team=self.team,
+ task=self.task,
+ state={
+ "startup_started_at": (self.started_at + timedelta(days=1)).isoformat(),
+ "startup_completed_at": (self.started_at + timedelta(days=2)).isoformat(),
+ "startup_provider": "claude",
+ "startup_runtime": "gvisor",
+ },
+ )
+ reconcile_research_startups(self.team.id, "example/research")
+ assert HealthIssue.objects.get(team=self.team, kind=ISSUE_KIND).status == HealthIssue.Status.ACTIVE
+
+ def test_other_team_cannot_create_an_issue_from_this_run(self) -> None:
+ notify_research_startup_failure(self.team.id + 10000, self.report_id, self.failure)
+ assert not HealthIssue.objects.filter(kind=ISSUE_KIND).exists()
+
+ def test_unknown_failure_does_not_claim_a_configuration_cause(self) -> None:
+ self.failure = replace(
+ self.failure,
+ failure=AgentStartupFailure(
+ provider="claude",
+ phase="sdk_initialization",
+ reason=None,
+ retryability="unknown",
+ ),
+ )
+ self.notify()
+ (event,) = stored_notifications_for_team(self.team.id)
+ assert "did not report a confirmed cause" in event.body
+ assert "automatically" not in event.body
+
+
[email protected]_db(transaction=True)
+def test_concurrent_failures_share_one_issue_and_notification() -> None:
+ organization = Organization.objects.create(name="Example concurrency")
+ team = Team.objects.create(organization=organization)
+ user = User.objects.create(email="[email protected]")
+ OrganizationMembership.objects.create(
+ organization=organization, user=user, level=OrganizationMembership.Level.ADMIN
+ )
+ task = Task.objects.create(team=team, created_by=user, repository="example/concurrent", title="Example research")
+ run = TaskRun.objects.create(
+ team=team, task=task, state={"startup_started_at": timezone.now().isoformat(), "startup_runtime": "gvisor"}
+ )
+ failure = ResearchStartupFailure(
+ failure=AgentStartupFailure(provider="claude", phase="sdk_initialization", reason=None, retryability="unknown"),
+ task_id=task.id,
+ run_id=run.id,
+ repository="example/concurrent",
+ )
+ barrier = Barrier(2)
+
+ def notify() -> None:
+ close_old_connections()
+ try:
+ barrier.wait(timeout=10)
+ notify_research_startup_failure(team.id, str(uuid4()), failure)
+ finally:
+ close_old_connections()
+
+ with (
+ patch("posthoganalytics.feature_enabled", return_value=True),
+ patch("products.notifications.backend.logic._publish_to_kafka"),
+ ThreadPoolExecutor(max_workers=2) as pool,
+ ):
+ futures = [pool.submit(notify) for _ in range(2)]
+ for future in futures:
+ future.result(timeout=30)
+ assert HealthIssue.objects.filter(team=team, kind=ISSUE_KIND).count() == 1
+ assert len(stored_notifications_for_team(team.id)) == 1
diff --git a/products/signals/backend/test/test_summary_workflow.py b/products/signals/backend/test/test_summary_workflow.py
index 1a5be7154ac7..7cc20e926a02 100644
--- a/products/signals/backend/test/test_summary_workflow.py
+++ b/products/signals/backend/test/test_summary_workflow.py
@@ -25,6 +25,7 @@ from products.signals.backend.models import SignalReport
from products.signals.backend.quota import SelfDrivingQuotaGate
from products.signals.backend.report_generation.research import ActionabilityChoice
from products.signals.backend.report_generation.select_repo import RepoSelectionResult
+from products.signals.backend.startup_failure import RESEARCH_STARTUP_FAILURE, ResearchStartupFailure
from products.signals.backend.temporal.agentic.report import (
RESEARCH_SANDBOX_UNAVAILABLE_ERROR_TYPE,
RunAgenticReportInput,
@@ -40,6 +41,7 @@ from products.signals.backend.temporal.summary import (
MarkReportInProgressInput,
MarkReportPendingInput,
MarkReportReadyInput,
+ NotifyResearchStartupFailureInput,
PublishReportCompletedInput,
ReportHasAssignedSignalsInput,
ResetReportToPotentialInput,
@@ -206,6 +208,8 @@ class _Recorder:
self.selected_repository = selected_repository
self.research_errors = list(research_errors or [])
self.failure_error_types: list[str | None] = []
+ self.failure_startups: list[ResearchStartupFailure | None] = []
+ self.startup_notice_attempts = 0
self.gate_checks: list[str] = []
self.fetches = 0
self.claimed_before_fetch: list[bool] = []
@@ -350,6 +354,15 @@ async def _run_summary_workflow(recorder: _Recorder) -> None:
recorder.failures += 1
recorder.failure_reasons.append(input.failure_reason)
recorder.failure_error_types.append(input.error_type)
+ recorder.failure_startups.append(input.startup_failure)
+
+ @activity.defn(name="notify_research_startup_failure_activity")
+ async def fake_startup_notification(input: NotifyResearchStartupFailureInput) -> None:
+ assert recorder.failures == 1
+ assert input.failure == recorder.failure_startups[0]
+ recorder.startup_notice_attempts += 1
+ if recorder.startup_notice_attempts == 1:
+ raise OperationalError("Notification temporarily unavailable")
# The production self-driving worker runs with the pydantic data converter; the default converter
# mangles the enum/pydantic payloads these activities exchange (RepoSelectionResult,
@@ -376,6 +389,7 @@ async def _run_summary_workflow(recorder: _Recorder) -> None:
fake_revert,
fake_reset,
fake_failed,
+ fake_startup_notification,
],
workflow_runner=UnsandboxedWorkflowRunner(),
):
@@ -469,6 +483,34 @@ async def test_research_does_not_retry_other_activity_errors(research_error: Exc
assert recorder.failure_error_types == [error_type]
[email protected]("requires_action,attempts", [(True, 1), (False, 2)])
+async def test_research_startup_retry_policy_and_failure_details_survive_temporal(requires_action: bool, attempts: int):
+ payload = {
+ "failure": {
+ "provider": "claude",
+ "phase": "sdk_initialization",
+ "reason": "proxy_invalid" if requires_action else None,
+ "retryability": "requires_action" if requires_action else "unknown",
+ },
+ "task_id": str(uuid.uuid4()),
+ "run_id": str(uuid.uuid4()),
+ "repository": "example/research",
+ }
+ error = ApplicationError(
+ "Research could not start", payload, type=RESEARCH_STARTUP_FAILURE, non_retryable=requires_action
+ )
+ recorder = _Recorder(research_errors=[error, error])
+ with pytest.raises(WorkflowFailureError):
+ await _run_summary_workflow(recorder)
+ assert recorder.researches == attempts
+ assert recorder.failures == 1
+ assert recorder.failure_startups[0] is not None
+ assert str(recorder.failure_startups[0].run_id) == payload["run_id"]
+ assert recorder.failure_startups[0].failure.retryability == ("requires_action" if requires_action else "unknown")
+ assert recorder.startup_notice_attempts == 2
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"choice,target",
diff --git a/products/signals/frontend/generated/api.schemas.ts b/products/signals/frontend/generated/api.schemas.ts
index 2b16ebc969e6..4c95b38d95a6 100644
--- a/products/signals/frontend/generated/api.schemas.ts
+++ b/products/signals/frontend/generated/api.schemas.ts
@@ -1530,6 +1530,77 @@ export const SignalReportStatusEnumApi = {
Suppressed: 'suppressed',
} as const
+/**
+ * * `claude` - Claude
+ * * `codex` - Codex
+ * * `pi` - Pi
+ */
+export type StartupProviderEnumApi = (typeof StartupProviderEnumApi)[keyof typeof StartupProviderEnumApi]
+
+export const StartupProviderEnumApi = {
+ Claude: 'claude',
+ Codex: 'codex',
+ Pi: 'pi',
+} as const
+
+/**
+ * * `session_setup` - Session setup
+ * * `sdk_initialization` - SDK initialization
+ * * `setup_hooks` - Setup hooks
+ */
+export type StartupPhaseEnumApi = (typeof StartupPhaseEnumApi)[keyof typeof StartupPhaseEnumApi]
+
+export const StartupPhaseEnumApi = {
+ SessionSetup: 'session_setup',
+ SdkInitialization: 'sdk_initialization',
+ SetupHooks: 'setup_hooks',
+} as const
+
+/**
+ * * `requires_action` - Requires action
+ * * `transient` - Transient
+ * * `unknown` - Unknown
+ */
+export type StartupRetryabilityEnumApi = (typeof StartupRetryabilityEnumApi)[keyof typeof StartupRetryabilityEnumApi]
+
+export const StartupRetryabilityEnumApi = {
+ RequiresAction: 'requires_action',
+ Transient: 'transient',
+ Unknown: 'unknown',
+} as const
+
+export interface ReportStartupFailureApi {
+ /** Agent provider that failed to start.
+ *
+ * * `claude` - Claude
+ * * `codex` - Codex
+ * * `pi` - Pi */
+ provider: StartupProviderEnumApi
+ /** Startup stage that failed.
+ *
+ * * `session_setup` - Session setup
+ * * `sdk_initialization` - SDK initialization
+ * * `setup_hooks` - Setup hooks */
+ phase: StartupPhaseEnumApi
+ /**
+ * Validated provider reason code, when available.
+ * @nullable
+ */
+ reason: string | null
+ /** Whether an unchanged startup can be retried.
+ *
+ * * `requires_action` - Requires action
+ * * `transient` - Transient
+ * * `unknown` - Unknown */
+ retryability: StartupRetryabilityEnumApi
+ /** Research task that could not start. */
+ task_id: string
+ /** Failed research task run. */
+ run_id: string
+ /** Safe explanation and suggested next action. */
+ message: string
+}
+
/**
* * `small` - small
* * `medium` - medium
@@ -2169,6 +2240,8 @@ export interface SignalReportListApi {
readonly signals_at_run: number
/** How many scout notes this report received beyond the few its work log keeps as entries. 0 when nothing was dropped. These say the finding still holds, so the count is shown in place of the entries. */
readonly collapsed_note_count: number
+ /** Startup failure from the latest failed research attempt, when available. */
+ readonly startup_failure: ReportStartupFailureApi | null
readonly created_at: string
readonly updated_at: string
readonly artefact_count: number
@@ -2421,6 +2494,8 @@ export interface SignalReportApi {
readonly signals_at_run: number
/** How many scout notes this report received beyond the few its work log keeps as entries. 0 when nothing was dropped. These say the finding still holds, so the count is shown in place of the entries. */
readonly collapsed_note_count: number
+ /** Startup failure from the latest failed research attempt, when available. */
+ readonly startup_failure: ReportStartupFailureApi | null
readonly created_at: string
readonly updated_at: string
readonly artefact_count: number
diff --git a/products/signals/frontend/inbox/components/detail/AgentRunDetail.tsx b/products/signals/frontend/inbox/components/detail/AgentRunDetail.tsx
index bb841ed0ed23..82473c279d1b 100644
--- a/products/signals/frontend/inbox/components/detail/AgentRunDetail.tsx
+++ b/products/signals/frontend/inbox/components/detail/AgentRunDetail.tsx
@@ -101,10 +101,12 @@ function RunOutputWidget({ report }: { report: SignalReport }): JSX.Element {
<IconWarning className="size-4" />
</span>
<div className="flex flex-col gap-0.5 min-w-0 flex-1">
- <span className="font-medium text-sm text-primary">Run failed</span>
+ <span className="font-medium text-sm text-primary">
+ {report.startup_failure ? 'Research could not start' : 'Run failed'}
+ </span>
<span className="text-xs text-secondary leading-snug">
- Research couldn't complete. Check the linked run below for the error. The agent may retry
- automatically.
+ {report.startup_failure?.message ??
+ "Research couldn't complete. Check the linked run below for the error before retrying."}
</span>
</div>
</div>
diff --git a/products/signals/frontend/inbox/types.ts b/products/signals/frontend/inbox/types.ts
index 0ca163f1f08e..21604aeda238 100644
--- a/products/signals/frontend/inbox/types.ts
+++ b/products/signals/frontend/inbox/types.ts
@@ -95,6 +95,7 @@ export const ACTIONABLE_ACTIONABILITY_VALUES: SignalReportActionability[] = [
]
export interface SignalReport extends Pick<SignalReportApi, 'monitoring_started_at'> {
+ startup_failure?: SignalReportApi['startup_failure']
pull_requests?: readonly SignalReportPullRequestApi[]
assignee?: SignalReportAssigneeApi | null
id: string
diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py
index 148dc2fee4e7..a2112f9a806a 100644
--- a/products/tasks/backend/facade/api.py
+++ b/products/tasks/backend/facade/api.py
@@ -45,6 +45,7 @@ from django.db.models import (
from django.db.models.fields.json import KeyTextTransform, KeyTransform
from django.db.models.functions import Coalesce
from django.utils import timezone as django_timezone
+from django.utils.dateparse import parse_datetime
from django.utils.http import content_disposition_header
import posthoganalytics
@@ -1133,6 +1134,30 @@ def get_task_run(run_id: str | UUID, team_id: int | None = None) -> contracts.Ta
return _task_run_to_dto(run)
+def latest_agent_startup_success(*, team_id: int, repository: str, provider: str, runtime: str) -> datetime | None:
+ started = (
+ TaskRun.objects.filter(
+ team_id=team_id,
+ state__has_key="agent_startup_success",
+ state__agent_startup_success__repository=repository.lower(),
+ state__agent_startup_success__provider=provider,
+ state__agent_startup_success__runtime=runtime,
+ )
+ .order_by("-state__agent_startup_success__started_at")
+ .values_list("state__agent_startup_success__started_at", flat=True)
+ .first()
+ )
+ if not isinstance(started, str):
+ return None
+ try:
+ started_at = parse_datetime(started)
+ except ValueError:
+ return None
+ if started_at is None or django_timezone.is_naive(started_at):
+ return None
+ return started_at
+
+
def find_signal_implementation_run(
*, team_id: int, repository: str, head_branch: str | None
) -> contracts.SignalImplementationRunDTO | None:
diff --git a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
index d8b589f75d65..83cf09b8a579 100644
--- a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
+++ b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py
@@ -1108,6 +1108,7 @@ class TestModalSandboxAgentServer:
assert mock_sandbox.execute.call_count == 1
def test_start_agent_server_skips_relaunch_when_already_healthy(self, mock_sandbox: Any):
+ before_launch = MagicMock()
mock_sandbox.execute = MagicMock(
return_value=ExecutionResult(stdout=_preflight_stdout(reused=True), stderr="", exit_code=0, error=None)
)
@@ -1123,9 +1124,11 @@ class TestModalSandboxAgentServer:
run_id="run-456",
mode="background",
allowed_domains=["example.com"],
+ before_launch=before_launch,
)
assert health_ms == 0
+ before_launch.assert_not_called()
wait_for_ready.assert_not_called()
mock_free.assert_not_called()
commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]
diff --git a/products/tasks/backend/migrations/0143_agent_startup_success_index.py b/products/tasks/backend/migrations/0143_agent_startup_success_index.py
new file mode 100644
index 000000000000..a4ac11d5f3b0
--- /dev/null
+++ b/products/tasks/backend/migrations/0143_agent_startup_success_index.py
@@ -0,0 +1,43 @@
+import django.db.models.fields.json
+from django.db import migrations, models
+
+from posthog.migration_helpers import SafeAddIndexConcurrently
+
+
+class Migration(migrations.Migration):
+ atomic = False
+
+ dependencies = [
+ ("posthog", "1414_remove_filesystem_rows_for_deleted_insights"),
+ ("tasks", "0142_task_profile_and_source"),
+ ]
+
+ operations = [
+ SafeAddIndexConcurrently(
+ model_name="taskrun",
+ index=models.Index(
+ models.F("team"),
+ django.db.models.fields.json.KeyTransform(
+ "repository",
+ django.db.models.fields.json.KeyTransform("agent_startup_success", "state"),
+ ),
+ django.db.models.fields.json.KeyTransform(
+ "provider",
+ django.db.models.fields.json.KeyTransform("agent_startup_success", "state"),
+ ),
+ django.db.models.fields.json.KeyTransform(
+ "runtime",
+ django.db.models.fields.json.KeyTransform("agent_startup_success", "state"),
+ ),
+ models.OrderBy(
+ django.db.models.fields.json.KeyTransform(
+ "started_at",
+ django.db.models.fields.json.KeyTransform("agent_startup_success", "state"),
+ ),
+ descending=True,
+ ),
+ condition=models.Q(("state__has_key", "agent_startup_success")),
+ name="task_run_startup_success_idx",
+ ),
+ ),
+ ]
diff --git a/products/tasks/backend/migrations/max_migration.txt b/products/tasks/backend/migrations/max_migration.txt
index 719d6408f792..606ea9bc34bf 100644
--- a/products/tasks/backend/migrations/max_migration.txt
+++ b/products/tasks/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0142_task_profile_and_source
+0143_agent_startup_success_index
diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py
index bda5ba5e84e3..3228ae3d6fb9 100644
--- a/products/tasks/backend/models.py
+++ b/products/tasks/backend/models.py
@@ -2376,6 +2376,15 @@ class TaskRun(models.Model):
db_table = "posthog_task_run"
ordering = ["-created_at"]
indexes = [
+ models.Index(
+ models.F("team"),
+ KeyTransform("repository", KeyTransform("agent_startup_success", "state")),
+ KeyTransform("provider", KeyTransform("agent_startup_success", "state")),
+ KeyTransform("runtime", KeyTransform("agent_startup_success", "state")),
+ KeyTransform("started_at", KeyTransform("agent_startup_success", "state")).desc(),
+ name="task_run_startup_success_idx",
+ condition=models.Q(state__has_key="agent_startup_success"),
+ ),
GinIndex(
OpClass(KeyTransform("verified_pr_urls", "state"), name="jsonb_path_ops"),
name="task_run_verified_pr_urls_idx",
diff --git a/products/tasks/backend/tests/test_startup_recovery.py b/products/tasks/backend/tests/test_startup_recovery.py
index 576a2351f0ed..b3884f53acaa 100644
--- a/products/tasks/backend/tests/test_startup_recovery.py
+++ b/products/tasks/backend/tests/test_startup_recovery.py
@@ -1,4 +1,5 @@
from datetime import timedelta
+from functools import partial
import time_machine
from posthog.test.base import BaseTest
@@ -7,6 +8,7 @@ from django.utils import timezone
from parameterized import parameterized
+from products.tasks.backend.facade.api import latest_agent_startup_success
from products.tasks.backend.models import Task, TaskRun
from products.tasks.backend.temporal.process_task.activities.get_task_processing_context import TaskProcessingContext
from products.tasks.backend.temporal.process_task.activities.start_agent_server import (
@@ -43,21 +45,31 @@ class TestAgentStartupRecovery(BaseTest):
task_runtime=runtime,
state={"runtime_adapter": adapter} if adapter else {},
)
+ latest_startup = partial(
+ latest_agent_startup_success,
+ team_id=self.team.id,
+ repository="example/research",
+ provider=provider,
+ runtime="gvisor",
+ )
started_at = timezone.now()
with time_machine.travel(started_at, tick=False):
_record_agent_startup_attempt(context, "gvisor")
run.refresh_from_db()
assert "agent_startup_success" not in run.state
+ assert latest_startup() is None
with time_machine.travel(started_at + timedelta(seconds=1), tick=False):
_record_agent_startup_success(context)
run.refresh_from_db()
success = run.state["agent_startup_success"]
assert success["started_at"] == started_at.isoformat()
assert success["provider"] == provider
+ assert latest_startup() == started_at
with time_machine.travel(started_at + timedelta(seconds=2), tick=False):
_record_agent_startup_attempt(context, "gvisor")
run.refresh_from_db()
assert run.state["agent_startup_success"] == success
+ assert latest_startup() == started_at
TaskRun.update_state_atomic(str(run.id), updates={"startup_failure": {"reason": "proxy_invalid"}})
TaskRun.objects.filter(id=run.id).update(error_message="Turn failed", status=final_status)
with time_machine.travel(started_at + timedelta(seconds=3), tick=False):
@@ -66,3 +78,4 @@ class TestAgentStartupRecovery(BaseTest):
assert "startup_failure" not in run.state
assert run.error_message == ("Turn failed" if run.is_terminal else None)
assert run.state["agent_startup_success"]["started_at"] == (started_at + timedelta(seconds=2)).isoformat()
+ assert latest_startup() == started_at + timedelta(seconds=2)
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 006788236614..da8825d83e46 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -79809,6 +79809,80 @@ export namespace Schemas {
Suppressed: 'suppressed',
} as const;
+ /**
+ * * `claude` - Claude
+ * * `codex` - Codex
+ * * `pi` - Pi
+ */
+ export type StartupProviderEnum = typeof StartupProviderEnum[keyof typeof StartupProviderEnum];
+
+
+ export const StartupProviderEnum = {
+ Claude: 'claude',
+ Codex: 'codex',
+ Pi: 'pi',
+ } as const;
+
+ /**
+ * * `session_setup` - Session setup
+ * * `sdk_initialization` - SDK initialization
+ * * `setup_hooks` - Setup hooks
+ */
+ export type StartupPhaseEnum = typeof StartupPhaseEnum[keyof typeof StartupPhaseEnum];
+
+
+ export const StartupPhaseEnum = {
+ SessionSetup: 'session_setup',
+ SdkInitialization: 'sdk_initialization',
+ SetupHooks: 'setup_hooks',
+ } as const;
+
+ /**
+ * * `requires_action` - Requires action
+ * * `transient` - Transient
+ * * `unknown` - Unknown
+ */
+ export type StartupRetryabilityEnum = typeof StartupRetryabilityEnum[keyof typeof StartupRetryabilityEnum];
+
+
+ export const StartupRetryabilityEnum = {
+ RequiresAction: 'requires_action',
+ Transient: 'transient',
+ Unknown: 'unknown',
+ } as const;
+
+ export interface ReportStartupFailure {
+ /** Agent provider that failed to start.
+ *
+ * * `claude` - Claude
+ * * `codex` - Codex
+ * * `pi` - Pi */
+ provider: StartupProviderEnum;
+ /** Startup stage that failed.
+ *
+ * * `session_setup` - Session setup
+ * * `sdk_initialization` - SDK initialization
+ * * `setup_hooks` - Setup hooks */
+ phase: StartupPhaseEnum;
+ /**
+ * Validated provider reason code, when available.
+ * @nullable
+ */
+ reason: string | null;
+ /** Whether an unchanged startup can be retried.
+ *
+ * * `requires_action` - Requires action
+ * * `transient` - Transient
+ * * `unknown` - Unknown */
+ retryability: StartupRetryabilityEnum;
+ /** Research task that could not start. */
+ task_id: string;
+ /** Failed research task run. */
+ run_id: string;
+ /** Safe explanation and suggested next action. */
+ message: string;
+ }
+
/**
* Snapshot-only metric shape for report lists.
*
@@ -80315,6 +80389,8 @@ export namespace Schemas {
readonly signals_at_run: number;
/** How many scout notes this report received beyond the few its work log keeps as entries. 0 when nothing was dropped. These say the finding still holds, so the count is shown in place of the entries. */
readonly collapsed_note_count: number;
+ /** Startup failure from the latest failed research attempt, when available. */
+ readonly startup_failure: ReportStartupFailure | null;
readonly created_at: string;
readonly updated_at: string;
readonly artefact_count: number;
@@ -103153,6 +103229,8 @@ export namespace Schemas {
readonly signals_at_run: number;
/** How many scout notes this report received beyond the few its work log keeps as entries. 0 when nothing was dropped. These say the finding still holds, so the count is shown in place of the entries. */
readonly collapsed_note_count: number;
+ /** Startup failure from the latest failed research attempt, when available. */
+ readonly startup_failure: ReportStartupFailure | null;
readonly created_at: string;
readonly updated_at: string;
readonly artefact_count: number;