Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): distinguish repository selection attempts and outcomes (#128141)

Georgiy Tarasoveeb26026271b
diff --git a/products/signals/backend/report_generation/select_repo.py b/products/signals/backend/report_generation/select_repo.py
index 1e6a6f03d6e9..13fd37cb170c 100644
--- a/products/signals/backend/report_generation/select_repo.py
+++ b/products/signals/backend/report_generation/select_repo.py
@@ -12,6 +12,7 @@ nowhere to land; `summary.py` treats `repository=None` as
from __future__ import annotations
import logging
+from collections.abc import Callable
from typing import TYPE_CHECKING
from posthog.egress.limiter.policies import Priority
@@ -78,6 +79,7 @@ async def select_repository_for_team(
sandbox_environment_id: str | None = None,
verbose: bool = False,
output_fn: OutputFn = None,
+ on_research_session: Callable[[str, str], None] | None = None,
source_repository: str | None = None,
) -> RepoSelectionResult:
"""Select the most relevant repository for a free-form request against the team's repos.
@@ -113,6 +115,7 @@ async def select_repository_for_team(
sandbox_environment_id=sandbox_environment_id,
verbose=verbose,
output_fn=output_fn,
+ on_research_session=on_research_session,
model=agent_runtime.model,
runtime_adapter=agent_runtime.runtime_adapter,
reasoning_effort=agent_runtime.reasoning_effort,
@@ -150,6 +153,7 @@ async def select_repository_for_report(
sandbox_environment_id: str | None = None,
verbose: bool = False,
output_fn: OutputFn = None,
+ on_research_session: Callable[[str, str], None] | None = None,
) -> RepoSelectionResult:
"""Select the most relevant repository for a set of signals.
@@ -170,5 +174,6 @@ async def select_repository_for_report(
sandbox_environment_id=sandbox_environment_id,
verbose=verbose,
output_fn=output_fn,
+ on_research_session=on_research_session,
source_repository=source_repository_from_signals(signals),
)
diff --git a/products/signals/backend/temporal/agentic/select_repository.py b/products/signals/backend/temporal/agentic/select_repository.py
index 3f560589d7a4..4050447b052b 100644
--- a/products/signals/backend/temporal/agentic/select_repository.py
+++ b/products/signals/backend/temporal/agentic/select_repository.py
@@ -32,7 +32,15 @@ from products.signals.backend.temporal.agentic import (
)
from products.signals.backend.temporal.types import SignalData
from products.tasks.backend.facade import api as tasks_facade
-from products.tasks.backend.facade.agents import RepositoryContextUnavailable
+from products.tasks.backend.facade.agents import (
+ AgentTurnFailed,
+ EmptyAgentTurnError,
+ RepositoryContextUnavailable,
+ SandboxRunFailed,
+ TruncatedAgentOutputError,
+ TurnPollTimeout,
+)
+from products.tasks.backend.facade.repo_selection import RepoSelectionUsageLimitError
from products.tasks.backend.facade.repo_selection_types import ReportResearchContext
# Repo discovery only runs `gh` CLI commands — limit egress to GitHub hosts.
@@ -89,6 +97,44 @@ def _resolve_sandbox_user_id(team_id: int) -> int | None:
return resolve_user_id_for_team(team_id, github=github)
+def _repo_selection_failure_reason(error: Exception, *, agent_started: bool) -> str:
+ if isinstance(error, RepoSelectionUsageLimitError):
+ return "usage_limit"
+ if isinstance(error, TurnPollTimeout):
+ return "poll_timeout"
+ if isinstance(error, EmptyAgentTurnError):
+ return "empty_output"
+ if isinstance(error, TruncatedAgentOutputError):
+ return "truncated_output"
+ if isinstance(error, ValueError) and agent_started:
+ return "output_parse_failed"
+ if isinstance(error, AgentTurnFailed):
+ return "agent_turn_failed"
+ if isinstance(error, SandboxRunFailed):
+ return "sandbox_failed"
+ if isinstance(error, (GitHubIntegrationError, GitHubRateLimitError, EgressBudgetExhausted, RequestException)):
+ return "github_unavailable"
+ return "agentic_activity_error"
+
+
+def _repo_research_attempt_properties(result: str | None, error: Exception | None) -> dict[str, object]:
+ properties: dict[str, object] = {}
+ if temporalio.activity.in_activity():
+ info = temporalio.activity.info()
+ properties.update(
+ workflow_id=info.workflow_id,
+ workflow_run_id=info.workflow_run_id,
+ activity_id=info.activity_id,
+ attempt=info.attempt,
+ )
+ if result is not None:
+ maximum_attempts = info.retry_policy.maximum_attempts if info.retry_policy else 0
+ permanent = isinstance(error, GitHubIntegrationError) and error.status_code in {401, 403, 404, 410}
+ if result != "failed" or permanent or maximum_attempts:
+ properties["is_final_attempt"] = result != "failed" or permanent or info.attempt >= maximum_attempts
+ return properties
+
+
def _capture_repo_research_event(
event: str,
team: Team,
@@ -96,8 +142,23 @@ def _capture_repo_research_event(
report_id: str,
result: str | None = None,
failure_reason: str | None = None,
+ *,
+ task_id: str | None = None,
+ task_run_id: str | None = None,
+ error: Exception | None = None,
) -> None:
- properties: dict = {"report_id": report_id}
+ properties: dict[str, object] = {"report_id": report_id, "event_scope": "attempt"}
+ properties.update(_repo_research_attempt_properties(result, error))
+ if task_id is not None:
+ properties["task_id"] = task_id
+ if task_run_id is not None:
+ properties["task_run_id"] = task_run_id
+ if error is not None:
+ properties["error_type"] = type(error).__name__
+ if isinstance(error, TurnPollTimeout):
+ properties.update(error.diagnostics())
+ if isinstance(error, AgentTurnFailed):
+ properties["agent_error_category"] = error.category
if result is not None:
properties["result"] = result
if failure_reason is not None:
@@ -141,6 +202,11 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelect
team.organization,
input.report_id,
)
+ research_ids: dict[str, str] = {}
+
+ def capture_research_session(task_id: str, run_id: str) -> None:
+ research_ids.update(task_id=task_id, task_run_id=run_id)
+
try:
async with Heartbeater():
# Check for a previous selection from an earlier run, if any
@@ -233,6 +299,7 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelect
signal_report_id=input.report_id,
spend_signal_id=input.signal_id,
sandbox_environment_id=sandbox_env_id,
+ on_research_session=capture_research_session,
)
# The shared selector takes a sole eligible repository without reading corrections.
# A recurrence must honor its parent's rejection before spending on research there.
@@ -258,6 +325,8 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelect
team.organization,
input.report_id,
result="selected" if result.repository is not None else "no_repo",
+ task_id=research_ids.get("task_id"),
+ task_run_id=research_ids.get("task_run_id"),
)
if input.github_optional:
availability = (
@@ -298,6 +367,9 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelect
input.report_id,
result="data_only",
failure_reason="github_unavailable",
+ error=e,
+ task_id=research_ids.get("task_id"),
+ task_run_id=research_ids.get("task_run_id"),
)
return RepoSelectionResult(
repository=None,
@@ -311,7 +383,10 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelect
team.organization,
input.report_id,
result="failed",
- failure_reason="agentic_activity_error",
+ failure_reason=_repo_selection_failure_reason(e, agent_started=bool(research_ids)),
+ error=e,
+ task_id=research_ids.get("task_id"),
+ task_run_id=research_ids.get("task_run_id"),
)
# Permanent GitHub App auth failures (installation gone/suspended) won't recover via retry.
if isinstance(e, GitHubIntegrationError) and e.status_code in {401, 403, 404, 410}:
diff --git a/products/signals/backend/temporal/summary.py b/products/signals/backend/temporal/summary.py
index 9904de689a53..fdcddcbad11c 100644
--- a/products/signals/backend/temporal/summary.py
+++ b/products/signals/backend/temporal/summary.py
@@ -147,6 +147,9 @@ async def _capture_report_event(
"run_count": run_count,
"source_products": source_products,
}
+ if temporalio.activity.in_activity():
+ info = temporalio.activity.info()
+ properties.update(workflow_id=info.workflow_id, workflow_run_id=info.workflow_run_id)
if result is not None:
properties["result"] = result
# Only the two outcomes that write prose carry a chart set, so the property is absent rather
diff --git a/products/signals/backend/test/test_agentic_report_activity.py b/products/signals/backend/test/test_agentic_report_activity.py
index a8d60c41ba1b..028ebe8e27d6 100644
--- a/products/signals/backend/test/test_agentic_report_activity.py
+++ b/products/signals/backend/test/test_agentic_report_activity.py
@@ -675,6 +675,7 @@ async def test_select_repository_activity_returns_repo(monkeypatch, ateam):
)
async def fake_select_repo(*args, **kwargs):
+ kwargs["on_research_session"]("example-task", "example-run")
return RepoSelectionResult(repository="posthog/posthog", reason="Single repository connected: posthog/posthog")
monkeypatch.setattr(
@@ -682,7 +683,10 @@ async def test_select_repository_activity_returns_repo(monkeypatch, ateam):
fake_select_repo,
)
- with patch("products.signals.backend.temporal.agentic.select_repository.Heartbeater"):
+ with (
+ patch("products.signals.backend.temporal.agentic.select_repository.Heartbeater"),
+ patch("products.signals.backend.temporal.agentic.select_repository.posthoganalytics.capture") as capture,
+ ):
result = await select_repository_activity(
SelectRepositoryInput(team_id=ateam.id, report_id=str(uuid4()), signals=_build_signals())
)
@@ -690,6 +694,12 @@ async def test_select_repository_activity_returns_repo(monkeypatch, ateam):
assert result.repository == "posthog/posthog"
assert "Single repository" in result.reason
+ completed = next(
+ call.kwargs for call in capture.call_args_list if call.kwargs["event"] == "signals_repo_research_completed"
+ )
+ assert completed["properties"]["task_id"] == "example-task"
+ assert completed["properties"]["task_run_id"] == "example-run"
+
@pytest.mark.asyncio
@pytest.mark.django_db
diff --git a/products/signals/backend/test/test_repo_selection_telemetry.py b/products/signals/backend/test/test_repo_selection_telemetry.py
new file mode 100644
index 000000000000..16a2c8ecd280
--- /dev/null
+++ b/products/signals/backend/test/test_repo_selection_telemetry.py
@@ -0,0 +1,60 @@
+from dataclasses import replace
+from functools import partial
+
+import pytest
+from unittest.mock import MagicMock, patch
+
+from temporalio.common import RetryPolicy
+from temporalio.testing import ActivityEnvironment
+
+from products.signals.backend.temporal.agentic.select_repository import (
+ _capture_repo_research_event,
+ _repo_selection_failure_reason,
+)
+from products.tasks.backend.facade.agents import EmptyAgentTurnError, SandboxRunFailed
+
+
+ "attempt,result,error,reason,final",
+ [
+ (1, "failed", ValueError("private model output"), "output_parse_failed", False),
+ (2, "failed", ValueError("private model output"), "output_parse_failed", True),
+ (1, "failed", EmptyAgentTurnError("empty", total_lines=0, printed_lines=0), "empty_output", False),
+ (1, "failed", SandboxRunFailed("unavailable", error_type=None), "sandbox_failed", False),
+ (2, "selected", None, None, True),
+ (1, "no_repo", None, None, True),
+ ],
+)
+def test_attempt_telemetry_distinguishes_retries_from_final_outcomes(attempt, result, error, reason, final) -> None:
+ environment = ActivityEnvironment()
+ environment.info = replace(
+ environment.info,
+ attempt=attempt,
+ retry_policy=RetryPolicy(maximum_attempts=2),
+ workflow_run_id="example-workflow-run",
+ )
+ with (
+ patch("products.signals.backend.temporal.agentic.select_repository.posthoganalytics.capture") as capture,
+ patch("products.signals.backend.temporal.agentic.select_repository.groups", return_value={}),
+ ):
+ environment.run(
+ partial(
+ _capture_repo_research_event,
+ "signals_repo_research_completed",
+ MagicMock(),
+ MagicMock(),
+ "example-report",
+ result=result,
+ failure_reason=_repo_selection_failure_reason(error, agent_started=True) if error else None,
+ error=error,
+ task_id="example-task",
+ task_run_id="example-run",
+ )
+ )
+ properties = capture.call_args.kwargs["properties"]
+ assert properties["workflow_run_id"] == "example-workflow-run"
+ assert properties["attempt"] == attempt
+ assert properties["is_final_attempt"] is final
+ assert properties.get("failure_reason") == reason
+ assert properties["task_run_id"] == "example-run"
+ assert "private model output" not in str(properties)
diff --git a/products/signals/backend/test/test_report_telemetry.py b/products/signals/backend/test/test_report_telemetry.py
index 3390d547a36e..1a5c48d33426 100644
--- a/products/signals/backend/test/test_report_telemetry.py
+++ b/products/signals/backend/test/test_report_telemetry.py
@@ -1,8 +1,11 @@
+from dataclasses import replace
from uuid import uuid4
import pytest
from unittest.mock import patch
+from temporalio.testing import ActivityEnvironment
+
from posthog.models import Team
from posthog.sync import database_sync_to_async
@@ -107,19 +110,23 @@ async def test_started_and_ready_fire_expected_captures(
report_id = str(report.id)
source_products = ["conversations", "zendesk"]
+ environment = ActivityEnvironment()
+ environment.info = replace(environment.info, workflow_run_id="example-workflow-run")
with patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture") as capture:
- await mark_report_in_progress_activity(
+ await environment.run(
+ mark_report_in_progress_activity,
MarkReportInProgressInput(
team_id=ateam.id,
report_id=report_id,
signal_count=2,
source_products=source_products,
preserve_pending_refresh=preserve_pending,
- )
+ ),
)
await database_sync_to_async(report.refresh_from_db)()
assert (report.pending_triggering_signal_id == driver) is preserve_pending
- await mark_report_ready_activity(
+ await environment.run(
+ mark_report_ready_activity,
MarkReportReadyInput(
team_id=ateam.id,
report_id=report_id,
@@ -130,7 +137,7 @@ async def test_started_and_ready_fire_expected_captures(
source_products=source_products,
charts=charts,
charts_enabled=charts_enabled,
- )
+ ),
)
# Model-level signal_report_status_changed labels ride the same analytics client; this test
@@ -140,6 +147,7 @@ async def test_started_and_ready_fire_expected_captures(
for e in events:
assert e["distinct_id"] == str(ateam.uuid)
assert e["properties"]["report_id"] == report_id
+ assert e["properties"]["workflow_run_id"] == "example-workflow-run"
assert e["properties"]["signal_count"] == 2
assert e["properties"]["source_products"] == source_products
assert "run_count" in e["properties"]
diff --git a/products/tasks/backend/logic/repo_selection/agent.py b/products/tasks/backend/logic/repo_selection/agent.py
index 9fcacb2d31ce..1850cf6939c9 100644
--- a/products/tasks/backend/logic/repo_selection/agent.py
+++ b/products/tasks/backend/logic/repo_selection/agent.py
@@ -23,7 +23,7 @@ from products.tasks.backend.logic.repo_selection.types import RepoSelectionResul
from products.tasks.backend.logic.services.custom_prompt_internals import CustomPromptSandboxContext
from products.tasks.backend.logic.services.custom_prompt_multi_turn_runner import MultiTurnSession
from products.tasks.backend.logic.services.sandbox import SandboxResources
-from products.tasks.backend.models import Task
+from products.tasks.backend.models import Task, TaskRun
if TYPE_CHECKING:
from products.tasks.backend.logic.services.custom_prompt_internals import OutputFn
@@ -767,6 +767,10 @@ async def select_repository(
raise ValueError("The end turn is a usage limit refusal, not a selection")
return _salvage_repo_selection(text, candidate_repos)
+ async def _session_created(task_run: TaskRun) -> None:
+ if on_research_session is not None:
+ on_research_session(str(task_run.task_id), str(task_run.id))
+
try:
session, result = await MultiTurnSession.start(
prompt=prompt,
@@ -780,6 +784,7 @@ async def select_repository(
spend_signal_id=spend_signal_id,
ai_stage="repo_selection",
internal=True,
+ on_task_run_created=_session_created,
fallback_from_text=_salvage,
json_retry_prompt=REPO_SELECTION_JSON_RETRY_PROMPT,
)
@@ -787,9 +792,6 @@ async def select_repository(
if usage_limit_reply is not None:
raise RepoSelectionUsageLimitError(usage_limit_reply) from e
raise
- # Track repo discovery execution (for example, for Slack)
- if on_research_session is not None:
- on_research_session(str(session.task.id), str(session.task_run.id))
# Stamp the producing task onto the result (overwriting anything the LLM may have emitted)
# so downstream persistence can attribute the selection to it.
result.task_id = str(session.task.id)
diff --git a/products/tasks/backend/tests/test_repo_selection_prompt.py b/products/tasks/backend/tests/test_repo_selection_prompt.py
index f3f9b6392cd5..3da3005f2d7e 100644
--- a/products/tasks/backend/tests/test_repo_selection_prompt.py
+++ b/products/tasks/backend/tests/test_repo_selection_prompt.py
@@ -275,10 +275,12 @@ def test_select_repository_asks_again_after_a_progress_note(
def test_select_repository_reports_a_usage_limit_refusal() -> None:
+ on_session = MagicMock()
github = MagicMock()
github.list_all_cached_repositories.return_value = [{"full_name": "acme/a"}, {"full_name": "acme/b"}]
async def start(**kwargs):
+ await kwargs["on_task_run_created"](MagicMock(task_id="example-task", id="example-run"))
# The runner swallows a raising fallback and raises its own parse error instead.
with pytest.raises(ValueError):
kwargs["fallback_from_text"]("API Error: 402 Your team has used its monthly PostHog AI credits.")
@@ -300,8 +302,11 @@ def test_select_repository_reports_a_usage_limit_refusal() -> None:
priority=Priority.NORMAL,
github=github,
candidate_repos=["acme/a", "acme/b"],
+ on_research_session=on_session,
)
+ on_session.assert_called_once_with("example-task", "example-run")
+
def test_source_repository_section_included_only_when_given() -> None:
base = _build_repo_selection_prompt("ctx", ["acme/a", "acme/b"])