:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signals): distinguish repository selection attempts and outcomes (#128141)
Georgiy Tarasoveeb26026271b
diff --git a/products/signals/backend/report_generation/select_repo.py b/products/signals/backend/report_generation/select_repo.pyindex 1e6a6f03d6e9..13fd37cb170c 100644--- a/products/signals/backend/report_generation/select_repo.py+++ b/products/signals/backend/report_generation/select_repo.py@@ -12,6 +12,7 @@ nowhere to land; `summary.py` treats `repository=None` asfrom __future__ import annotationsimport logging+from collections.abc import Callablefrom typing import TYPE_CHECKINGfrom posthog.egress.limiter.policies import Priority@@ -78,6 +79,7 @@ async def select_repository_for_team(sandbox_environment_id: str | None = None,verbose: bool = False,output_fn: OutputFn = None,+ on_research_session: Callable[[str, str], None] | None = None,source_repository: str | None = None,) -> RepoSelectionResult:"""Select the most relevant repository for a free-form request against the team's repos.@@ -113,6 +115,7 @@ async def select_repository_for_team(sandbox_environment_id=sandbox_environment_id,verbose=verbose,output_fn=output_fn,+ on_research_session=on_research_session,model=agent_runtime.model,runtime_adapter=agent_runtime.runtime_adapter,reasoning_effort=agent_runtime.reasoning_effort,@@ -150,6 +153,7 @@ async def select_repository_for_report(sandbox_environment_id: str | None = None,verbose: bool = False,output_fn: OutputFn = None,+ on_research_session: Callable[[str, str], None] | None = None,) -> RepoSelectionResult:"""Select the most relevant repository for a set of signals.@@ -170,5 +174,6 @@ async def select_repository_for_report(sandbox_environment_id=sandbox_environment_id,verbose=verbose,output_fn=output_fn,+ on_research_session=on_research_session,source_repository=source_repository_from_signals(signals),)diff --git a/products/signals/backend/temporal/agentic/select_repository.py b/products/signals/backend/temporal/agentic/select_repository.pyindex 3f560589d7a4..4050447b052b 100644--- a/products/signals/backend/temporal/agentic/select_repository.py+++ b/products/signals/backend/temporal/agentic/select_repository.py@@ -32,7 +32,15 @@ from products.signals.backend.temporal.agentic import ()from products.signals.backend.temporal.types import SignalDatafrom products.tasks.backend.facade import api as tasks_facade-from products.tasks.backend.facade.agents import RepositoryContextUnavailable+from products.tasks.backend.facade.agents import (+ AgentTurnFailed,+ EmptyAgentTurnError,+ RepositoryContextUnavailable,+ SandboxRunFailed,+ TruncatedAgentOutputError,+ TurnPollTimeout,+)+from products.tasks.backend.facade.repo_selection import RepoSelectionUsageLimitErrorfrom products.tasks.backend.facade.repo_selection_types import ReportResearchContext# Repo discovery only runs `gh` CLI commands — limit egress to GitHub hosts.@@ -89,6 +97,44 @@ def _resolve_sandbox_user_id(team_id: int) -> int | None:return resolve_user_id_for_team(team_id, github=github)+def _repo_selection_failure_reason(error: Exception, *, agent_started: bool) -> str:+ if isinstance(error, RepoSelectionUsageLimitError):+ return "usage_limit"+ if isinstance(error, TurnPollTimeout):+ return "poll_timeout"+ if isinstance(error, EmptyAgentTurnError):+ return "empty_output"+ if isinstance(error, TruncatedAgentOutputError):+ return "truncated_output"+ if isinstance(error, ValueError) and agent_started:+ return "output_parse_failed"+ if isinstance(error, AgentTurnFailed):+ return "agent_turn_failed"+ if isinstance(error, SandboxRunFailed):+ return "sandbox_failed"+ if isinstance(error, (GitHubIntegrationError, GitHubRateLimitError, EgressBudgetExhausted, RequestException)):+ return "github_unavailable"+ return "agentic_activity_error"+++def _repo_research_attempt_properties(result: str | None, error: Exception | None) -> dict[str, object]:+ properties: dict[str, object] = {}+ if temporalio.activity.in_activity():+ info = temporalio.activity.info()+ properties.update(+ workflow_id=info.workflow_id,+ workflow_run_id=info.workflow_run_id,+ activity_id=info.activity_id,+ attempt=info.attempt,+ )+ if result is not None:+ maximum_attempts = info.retry_policy.maximum_attempts if info.retry_policy else 0+ permanent = isinstance(error, GitHubIntegrationError) and error.status_code in {401, 403, 404, 410}+ if result != "failed" or permanent or maximum_attempts:+ properties["is_final_attempt"] = result != "failed" or permanent or info.attempt >= maximum_attempts+ return properties++def _capture_repo_research_event(event: str,team: Team,@@ -96,8 +142,23 @@ def _capture_repo_research_event(report_id: str,result: str | None = None,failure_reason: str | None = None,+ *,+ task_id: str | None = None,+ task_run_id: str | None = None,+ error: Exception | None = None,) -> None:- properties: dict = {"report_id": report_id}+ properties: dict[str, object] = {"report_id": report_id, "event_scope": "attempt"}+ properties.update(_repo_research_attempt_properties(result, error))+ if task_id is not None:+ properties["task_id"] = task_id+ if task_run_id is not None:+ properties["task_run_id"] = task_run_id+ if error is not None:+ properties["error_type"] = type(error).__name__+ if isinstance(error, TurnPollTimeout):+ properties.update(error.diagnostics())+ if isinstance(error, AgentTurnFailed):+ properties["agent_error_category"] = error.categoryif result is not None:properties["result"] = resultif failure_reason is not None:@@ -141,6 +202,11 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelectteam.organization,input.report_id,)+ research_ids: dict[str, str] = {}++ def capture_research_session(task_id: str, run_id: str) -> None:+ research_ids.update(task_id=task_id, task_run_id=run_id)+try:async with Heartbeater():# Check for a previous selection from an earlier run, if any@@ -233,6 +299,7 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelectsignal_report_id=input.report_id,spend_signal_id=input.signal_id,sandbox_environment_id=sandbox_env_id,+ on_research_session=capture_research_session,)# The shared selector takes a sole eligible repository without reading corrections.# A recurrence must honor its parent's rejection before spending on research there.@@ -258,6 +325,8 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelectteam.organization,input.report_id,result="selected" if result.repository is not None else "no_repo",+ task_id=research_ids.get("task_id"),+ task_run_id=research_ids.get("task_run_id"),)if input.github_optional:availability = (@@ -298,6 +367,9 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelectinput.report_id,result="data_only",failure_reason="github_unavailable",+ error=e,+ task_id=research_ids.get("task_id"),+ task_run_id=research_ids.get("task_run_id"),)return RepoSelectionResult(repository=None,@@ -311,7 +383,10 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelectteam.organization,input.report_id,result="failed",- failure_reason="agentic_activity_error",+ failure_reason=_repo_selection_failure_reason(e, agent_started=bool(research_ids)),+ error=e,+ task_id=research_ids.get("task_id"),+ task_run_id=research_ids.get("task_run_id"),)# Permanent GitHub App auth failures (installation gone/suspended) won't recover via retry.if isinstance(e, GitHubIntegrationError) and e.status_code in {401, 403, 404, 410}:diff --git a/products/signals/backend/temporal/summary.py b/products/signals/backend/temporal/summary.pyindex 9904de689a53..fdcddcbad11c 100644--- a/products/signals/backend/temporal/summary.py+++ b/products/signals/backend/temporal/summary.py@@ -147,6 +147,9 @@ async def _capture_report_event("run_count": run_count,"source_products": source_products,}+ if temporalio.activity.in_activity():+ info = temporalio.activity.info()+ properties.update(workflow_id=info.workflow_id, workflow_run_id=info.workflow_run_id)if result is not None:properties["result"] = result# Only the two outcomes that write prose carry a chart set, so the property is absent ratherdiff --git a/products/signals/backend/test/test_agentic_report_activity.py b/products/signals/backend/test/test_agentic_report_activity.pyindex a8d60c41ba1b..028ebe8e27d6 100644--- a/products/signals/backend/test/test_agentic_report_activity.py+++ b/products/signals/backend/test/test_agentic_report_activity.py@@ -675,6 +675,7 @@ async def test_select_repository_activity_returns_repo(monkeypatch, ateam):)async def fake_select_repo(*args, **kwargs):+ kwargs["on_research_session"]("example-task", "example-run")return RepoSelectionResult(repository="posthog/posthog", reason="Single repository connected: posthog/posthog")monkeypatch.setattr(@@ -682,7 +683,10 @@ async def test_select_repository_activity_returns_repo(monkeypatch, ateam):fake_select_repo,)- with patch("products.signals.backend.temporal.agentic.select_repository.Heartbeater"):+ with (+ patch("products.signals.backend.temporal.agentic.select_repository.Heartbeater"),+ patch("products.signals.backend.temporal.agentic.select_repository.posthoganalytics.capture") as capture,+ ):result = await select_repository_activity(SelectRepositoryInput(team_id=ateam.id, report_id=str(uuid4()), signals=_build_signals()))@@ -690,6 +694,12 @@ async def test_select_repository_activity_returns_repo(monkeypatch, ateam):assert result.repository == "posthog/posthog"assert "Single repository" in result.reason+ completed = next(+ call.kwargs for call in capture.call_args_list if call.kwargs["event"] == "signals_repo_research_completed"+ )+ assert completed["properties"]["task_id"] == "example-task"+ assert completed["properties"]["task_run_id"] == "example-run"+@pytest.mark.asyncio@pytest.mark.django_dbdiff --git a/products/signals/backend/test/test_repo_selection_telemetry.py b/products/signals/backend/test/test_repo_selection_telemetry.pynew file mode 100644index 000000000000..16a2c8ecd280--- /dev/null+++ b/products/signals/backend/test/test_repo_selection_telemetry.py@@ -0,0 +1,60 @@+from dataclasses import replace+from functools import partial++import pytest+from unittest.mock import MagicMock, patch++from temporalio.common import RetryPolicy+from temporalio.testing import ActivityEnvironment++from products.signals.backend.temporal.agentic.select_repository import (+ _capture_repo_research_event,+ _repo_selection_failure_reason,+)+from products.tasks.backend.facade.agents import EmptyAgentTurnError, SandboxRunFailed+++ "attempt,result,error,reason,final",+ [+ (1, "failed", ValueError("private model output"), "output_parse_failed", False),+ (2, "failed", ValueError("private model output"), "output_parse_failed", True),+ (1, "failed", EmptyAgentTurnError("empty", total_lines=0, printed_lines=0), "empty_output", False),+ (1, "failed", SandboxRunFailed("unavailable", error_type=None), "sandbox_failed", False),+ (2, "selected", None, None, True),+ (1, "no_repo", None, None, True),+ ],+)+def test_attempt_telemetry_distinguishes_retries_from_final_outcomes(attempt, result, error, reason, final) -> None:+ environment = ActivityEnvironment()+ environment.info = replace(+ environment.info,+ attempt=attempt,+ retry_policy=RetryPolicy(maximum_attempts=2),+ workflow_run_id="example-workflow-run",+ )+ with (+ patch("products.signals.backend.temporal.agentic.select_repository.posthoganalytics.capture") as capture,+ patch("products.signals.backend.temporal.agentic.select_repository.groups", return_value={}),+ ):+ environment.run(+ partial(+ _capture_repo_research_event,+ "signals_repo_research_completed",+ MagicMock(),+ MagicMock(),+ "example-report",+ result=result,+ failure_reason=_repo_selection_failure_reason(error, agent_started=True) if error else None,+ error=error,+ task_id="example-task",+ task_run_id="example-run",+ )+ )+ properties = capture.call_args.kwargs["properties"]+ assert properties["workflow_run_id"] == "example-workflow-run"+ assert properties["attempt"] == attempt+ assert properties["is_final_attempt"] is final+ assert properties.get("failure_reason") == reason+ assert properties["task_run_id"] == "example-run"+ assert "private model output" not in str(properties)diff --git a/products/signals/backend/test/test_report_telemetry.py b/products/signals/backend/test/test_report_telemetry.pyindex 3390d547a36e..1a5c48d33426 100644--- a/products/signals/backend/test/test_report_telemetry.py+++ b/products/signals/backend/test/test_report_telemetry.py@@ -1,8 +1,11 @@+from dataclasses import replacefrom uuid import uuid4import pytestfrom unittest.mock import patch+from temporalio.testing import ActivityEnvironment+from posthog.models import Teamfrom posthog.sync import database_sync_to_async@@ -107,19 +110,23 @@ async def test_started_and_ready_fire_expected_captures(report_id = str(report.id)source_products = ["conversations", "zendesk"]+ environment = ActivityEnvironment()+ environment.info = replace(environment.info, workflow_run_id="example-workflow-run")with patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture") as capture:- await mark_report_in_progress_activity(+ await environment.run(+ mark_report_in_progress_activity,MarkReportInProgressInput(team_id=ateam.id,report_id=report_id,signal_count=2,source_products=source_products,preserve_pending_refresh=preserve_pending,- )+ ),)await database_sync_to_async(report.refresh_from_db)()assert (report.pending_triggering_signal_id == driver) is preserve_pending- await mark_report_ready_activity(+ await environment.run(+ mark_report_ready_activity,MarkReportReadyInput(team_id=ateam.id,report_id=report_id,@@ -130,7 +137,7 @@ async def test_started_and_ready_fire_expected_captures(source_products=source_products,charts=charts,charts_enabled=charts_enabled,- )+ ),)# Model-level signal_report_status_changed labels ride the same analytics client; this test@@ -140,6 +147,7 @@ async def test_started_and_ready_fire_expected_captures(for e in events:assert e["distinct_id"] == str(ateam.uuid)assert e["properties"]["report_id"] == report_id+ assert e["properties"]["workflow_run_id"] == "example-workflow-run"assert e["properties"]["signal_count"] == 2assert e["properties"]["source_products"] == source_productsassert "run_count" in e["properties"]diff --git a/products/tasks/backend/logic/repo_selection/agent.py b/products/tasks/backend/logic/repo_selection/agent.pyindex 9fcacb2d31ce..1850cf6939c9 100644--- a/products/tasks/backend/logic/repo_selection/agent.py+++ b/products/tasks/backend/logic/repo_selection/agent.py@@ -23,7 +23,7 @@ from products.tasks.backend.logic.repo_selection.types import RepoSelectionResulfrom products.tasks.backend.logic.services.custom_prompt_internals import CustomPromptSandboxContextfrom products.tasks.backend.logic.services.custom_prompt_multi_turn_runner import MultiTurnSessionfrom products.tasks.backend.logic.services.sandbox import SandboxResources-from products.tasks.backend.models import Task+from products.tasks.backend.models import Task, TaskRunif TYPE_CHECKING:from products.tasks.backend.logic.services.custom_prompt_internals import OutputFn@@ -767,6 +767,10 @@ async def select_repository(raise ValueError("The end turn is a usage limit refusal, not a selection")return _salvage_repo_selection(text, candidate_repos)+ async def _session_created(task_run: TaskRun) -> None:+ if on_research_session is not None:+ on_research_session(str(task_run.task_id), str(task_run.id))+try:session, result = await MultiTurnSession.start(prompt=prompt,@@ -780,6 +784,7 @@ async def select_repository(spend_signal_id=spend_signal_id,ai_stage="repo_selection",internal=True,+ on_task_run_created=_session_created,fallback_from_text=_salvage,json_retry_prompt=REPO_SELECTION_JSON_RETRY_PROMPT,)@@ -787,9 +792,6 @@ async def select_repository(if usage_limit_reply is not None:raise RepoSelectionUsageLimitError(usage_limit_reply) from eraise- # Track repo discovery execution (for example, for Slack)- if on_research_session is not None:- on_research_session(str(session.task.id), str(session.task_run.id))# Stamp the producing task onto the result (overwriting anything the LLM may have emitted)# so downstream persistence can attribute the selection to it.result.task_id = str(session.task.id)diff --git a/products/tasks/backend/tests/test_repo_selection_prompt.py b/products/tasks/backend/tests/test_repo_selection_prompt.pyindex f3f9b6392cd5..3da3005f2d7e 100644--- a/products/tasks/backend/tests/test_repo_selection_prompt.py+++ b/products/tasks/backend/tests/test_repo_selection_prompt.py@@ -275,10 +275,12 @@ def test_select_repository_asks_again_after_a_progress_note(def test_select_repository_reports_a_usage_limit_refusal() -> None:+ on_session = MagicMock()github = MagicMock()github.list_all_cached_repositories.return_value = [{"full_name": "acme/a"}, {"full_name": "acme/b"}]async def start(**kwargs):+ await kwargs["on_task_run_created"](MagicMock(task_id="example-task", id="example-run"))# The runner swallows a raising fallback and raises its own parse error instead.with pytest.raises(ValueError):kwargs["fallback_from_text"]("API Error: 402 Your team has used its monthly PostHog AI credits.")@@ -300,8 +302,11 @@ def test_select_repository_reports_a_usage_limit_refusal() -> None:priority=Priority.NORMAL,github=github,candidate_repos=["acme/a", "acme/b"],+ on_research_session=on_session,)+ on_session.assert_called_once_with("example-task", "example-run")+def test_source_repository_section_included_only_when_given() -> None:base = _build_repo_selection_prompt("ctx", ["acme/a", "acme/b"])