:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): discover eval sampler teams from the run's window (#122573)
Bernat Torres46fc6960d679
diff --git a/posthog/temporal/ai_observability/evaluation_clustering/coordinator.py b/posthog/temporal/ai_observability/evaluation_clustering/coordinator.pyindex e9a4c431e795..6bb7a150e62e 100644--- a/posthog/temporal/ai_observability/evaluation_clustering/coordinator.py+++ b/posthog/temporal/ai_observability/evaluation_clustering/coordinator.py@@ -26,6 +26,8 @@ from posthog.temporal.ai_observability.evaluation_clustering.models import (from posthog.temporal.ai_observability.evaluation_clustering.workflow import (AIObservabilityEvaluationClusteringWorkflow,AIObservabilityEvaluationSamplerWorkflow,+ SamplerWindow,+ sampler_window,)from posthog.temporal.ai_observability.trace_clustering.constants import WORKFLOW_EXECUTION_TIMEOUTfrom posthog.temporal.common.base import PostHogWorkflow@@ -70,9 +72,10 @@ class _LevelCoordinatorInputs:results_so_far: dict[str, Any] | None = None[email protected](frozen=False)class SamplerCoordinatorInputs(_LevelCoordinatorInputs):- pass+ window_start: str | None = None+ window_end: str | None = None@dataclasses.dataclass@@ -90,12 +93,14 @@ def _empty_results() -> dict[str, Any]:}-async def _discover_teams_and_jobs() -> tuple[list[int], dict[int, list[JobConfig]]]:+async def _discover_teams_and_jobs(+ discovery: TeamDiscoveryInput | None = None,+) -> tuple[list[int], dict[int, list[JobConfig]]]:"""Team discovery + fetch all clustering jobs, reusing the shared LLMA activities."""try:team_ids = await workflow.execute_activity(get_team_ids_for_ai_observability,- TeamDiscoveryInput(),+ discovery or TeamDiscoveryInput(),start_to_close_timeout=DISCOVERY_ACTIVITY_TIMEOUT,retry_policy=DISCOVERY_ACTIVITY_RETRY_POLICY,)@@ -120,6 +125,14 @@ async def _discover_teams_and_jobs() -> tuple[list[int], dict[int, list[JobConfireturn team_ids, per_team_jobs+def _run_window(inputs: SamplerCoordinatorInputs) -> SamplerWindow:+ if inputs.window_start and inputs.window_end:+ return SamplerWindow(start=inputs.window_start, end=inputs.window_end)+ # workflow_start_time, not now(): a continue_as_new leg carries the window, and a run that a worker picks+ # up late must still sample the hour it was scheduled for.+ return sampler_window(workflow.info().workflow_start_time)++def _evaluation_jobs_for_team(team_jobs: list[JobConfig]) -> list[JobConfig]:"""Pick the evaluation-level jobs for a team.@@ -145,6 +158,8 @@ class AIObservabilityEvaluationSamplerCoordinatorWorkflow(PostHogWorkflow):@workflow.runasync def run(self, inputs: SamplerCoordinatorInputs) -> SamplerCoordinatorResult:+ window = _run_window(inputs)+ window_start, window_end = window.start, window.endif inputs.remaining_team_ids is not None:team_ids = inputs.remaining_team_idsper_team_jobs: dict[int, list[JobConfig]] = {}@@ -154,7 +169,11 @@ class AIObservabilityEvaluationSamplerCoordinatorWorkflow(PostHogWorkflow):results_so_far = inputs.results_so_far or _empty_results()else:logger.info("Starting evaluation sampler coordinator")- team_ids, per_team_jobs = await _discover_teams_and_jobs()+ # Most teams with AI events have no $ai_evaluation events in a given hour, and each one would cost a+ # child workflow and a sampling query that finds nothing. The event and window match the sampler query.+ team_ids, per_team_jobs = await _discover_teams_and_jobs(+ TeamDiscoveryInput(window_start=window_start, window_end=window_end, trigger_events=["$ai_evaluation"])+ )record_teams_discovered(len(team_ids), "eval_sampling", "evaluation")results_so_far = _empty_results()@@ -174,6 +193,8 @@ class AIObservabilityEvaluationSamplerCoordinatorWorkflow(PostHogWorkflow):job_id=job.job_id,job_name=job.name,event_filters=job.event_filters,+ window_start=window_start,+ window_end=window_end,),id=(f"{SAMPLER_CHILD_WORKFLOW_ID_PREFIX}-{team_id}-{job.job_id}-{workflow.now().isoformat()}"),execution_timeout=SAMPLER_WORKFLOW_EXECUTION_TIMEOUT,@@ -207,6 +228,8 @@ class AIObservabilityEvaluationSamplerCoordinatorWorkflow(PostHogWorkflow):remaining_team_ids=remaining,per_team_jobs=serializable_jobs,results_so_far=results_so_far,+ window_start=window_start,+ window_end=window_end,))diff --git a/posthog/temporal/ai_observability/evaluation_clustering/tests/test_workflow.py b/posthog/temporal/ai_observability/evaluation_clustering/tests/test_workflow.pyindex f430ffc979a0..a40c6048918a 100644--- a/posthog/temporal/ai_observability/evaluation_clustering/tests/test_workflow.py+++ b/posthog/temporal/ai_observability/evaluation_clustering/tests/test_workflow.py@@ -6,7 +6,8 @@ from datetime import UTC, datetime, timedeltaimport pytestfrom unittest.mock import patch-from temporalio import activity+from parameterized import parameterized+from temporalio import activity, workflowfrom temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Worker@@ -22,18 +23,25 @@ from posthog.temporal.ai_observability.evaluation_clustering.constants import (SAMPLER_MAX_SAMPLES_PER_JOB,SAMPLER_WINDOW_MINUTES,SAMPLER_WINDOW_OFFSET_MINUTES,+ SAMPLER_WORKFLOW_NAME,+)+from posthog.temporal.ai_observability.evaluation_clustering.coordinator import (+ AIObservabilityEvaluationSamplerCoordinatorWorkflow,+ SamplerCoordinatorInputs,+ _evaluation_jobs_for_team,)-from posthog.temporal.ai_observability.evaluation_clustering.coordinator import _evaluation_jobs_for_teamfrom posthog.temporal.ai_observability.evaluation_clustering.models import (SamplerActivityInputs,SamplerActivityResult,SamplerWorkflowInputs,+ SamplerWorkflowResult,)from posthog.temporal.ai_observability.evaluation_clustering.workflow import (AIObservabilityEvaluationClusteringWorkflow,AIObservabilityEvaluationSamplerWorkflow,)-from posthog.temporal.ai_observability.shared_activities import JobConfig+from posthog.temporal.ai_observability.shared_activities import FetchAllClusteringJobsInput, JobConfig+from posthog.temporal.ai_observability.team_discovery import TeamDiscoveryInputfrom posthog.temporal.ai_observability.trace_clustering.models import ClusteringMetrics, ClusteringResult, ClusterLabel@@ -188,6 +196,75 @@ class TestSamplerWorkflowWindowMath:assert activity_inputs.event_filters == filters+sampler_child_windows: list[tuple[str, str]] = []+sampler_discovery_inputs: list[TeamDiscoveryInput | None] = []++[email protected](name=SAMPLER_WORKFLOW_NAME)+class FakeEvaluationSamplerWorkflow:+ @workflow.run+ async def run(self, inputs: SamplerWorkflowInputs) -> SamplerWorkflowResult:+ if not workflow.unsafe.is_replaying():+ sampler_child_windows.append((inputs.window_start or "", inputs.window_end or ""))+ # A later continue_as_new leg then starts minutes after the first, so it would derive a different window.+ await workflow.sleep(timedelta(minutes=10))+ return SamplerWorkflowResult(+ team_id=inputs.team_id,+ job_id=inputs.job_id,+ sampled=0,+ embedded=0,+ window_start=inputs.window_start or "",+ window_end=inputs.window_end or "",+ )++[email protected](name="get_team_ids_for_llm_analytics")+async def fake_eval_team_discovery(inputs: TeamDiscoveryInput | None = None) -> list[int]:+ sampler_discovery_inputs.append(inputs)+ return [1, 2]++[email protected](name="fetch_all_clustering_jobs_activity")+async def fake_fetch_eval_jobs(inputs: FetchAllClusteringJobsInput) -> dict[int, list[JobConfig]]:+ return {+ team_id: [JobConfig(job_id=f"j{team_id}", name="evals", analysis_level="evaluation", event_filters=[])]+ for team_id in inputs.team_ids+ }+++class TestSamplerCoordinatorWindow:+ @parameterized.expand([("single_leg", False), ("continue_as_new_each_batch", True)])+ @pytest.mark.asyncio+ async def test_children_sample_the_window_discovery_checked(self, _name, continue_as_new):+ sampler_child_windows.clear()+ sampler_discovery_inputs.clear()+ task_queue = str(uuid.uuid4())+ with patch.object(workflow.Info, "is_continue_as_new_suggested", return_value=continue_as_new):+ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with Worker(+ env.client,+ task_queue=task_queue,+ workflows=[AIObservabilityEvaluationSamplerCoordinatorWorkflow, FakeEvaluationSamplerWorkflow],+ activities=[fake_eval_team_discovery, fake_fetch_eval_jobs],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ await env.client.execute_workflow(+ AIObservabilityEvaluationSamplerCoordinatorWorkflow.run,+ SamplerCoordinatorInputs(max_concurrent_teams=1),+ id=str(uuid.uuid4()),+ task_queue=task_queue,+ )++ assert len(sampler_child_windows) == 2+ windows = set(sampler_child_windows)+ assert len(windows) == 1+ window_start, window_end = next(iter(windows))+ assert _parse_z(window_end) - _parse_z(window_start) == timedelta(minutes=SAMPLER_WINDOW_MINUTES)+ assert sampler_discovery_inputs == [+ TeamDiscoveryInput(window_start=window_start, window_end=window_end, trigger_events=["$ai_evaluation"])+ ]++class TestClusteringWorkflowWiring:@pytest.mark.asyncioasync def test_forwards_workflow_run_identity_to_labeling(self) -> None:diff --git a/posthog/temporal/ai_observability/evaluation_clustering/workflow.py b/posthog/temporal/ai_observability/evaluation_clustering/workflow.pyindex 60af092f01ba..ab619816664b 100644--- a/posthog/temporal/ai_observability/evaluation_clustering/workflow.py+++ b/posthog/temporal/ai_observability/evaluation_clustering/workflow.py@@ -1,11 +1,12 @@"""Per-job workflows for evaluation clustering (Stage A sampler + Stage B clustering stub)."""-from datetime import timedelta+from datetime import datetime, timedeltaimport structlogimport temporalio.exceptionsfrom temporalio import workflow+from posthog.dataclasses import frozenfrom posthog.temporal.ai_observability.evaluation_clustering.constants import (CLUSTERING_WORKFLOW_NAME,SAMPLER_ACTIVITY_HEARTBEAT,@@ -33,6 +34,18 @@ from posthog.temporal.common.base import PostHogWorkflowlogger = structlog.get_logger(__name__)+@frozen+class SamplerWindow:+ start: str+ end: str+++def sampler_window(now: datetime) -> SamplerWindow:+ end = now - timedelta(minutes=SAMPLER_WINDOW_OFFSET_MINUTES)+ start = end - timedelta(minutes=SAMPLER_WINDOW_MINUTES)+ return SamplerWindow(start=start.strftime("%Y-%m-%dT%H:%M:%SZ"), end=end.strftime("%Y-%m-%dT%H:%M:%SZ"))++@workflow.defn(name=SAMPLER_WORKFLOW_NAME)class AIObservabilityEvaluationSamplerWorkflow(PostHogWorkflow):"""Hourly per-(team, eval ClusteringJob) sampler.@@ -55,15 +68,12 @@ class AIObservabilityEvaluationSamplerWorkflow(PostHogWorkflow):@workflow.runasync def run(self, inputs: SamplerWorkflowInputs) -> SamplerWorkflowResult:- now = workflow.now()if inputs.window_start and inputs.window_end:window_start = inputs.window_startwindow_end = inputs.window_endelse:- window_end_dt = now - timedelta(minutes=SAMPLER_WINDOW_OFFSET_MINUTES)- window_start_dt = window_end_dt - timedelta(minutes=SAMPLER_WINDOW_MINUTES)- window_start = window_start_dt.strftime("%Y-%m-%dT%H:%M:%SZ")- window_end = window_end_dt.strftime("%Y-%m-%dT%H:%M:%SZ")+ window = sampler_window(workflow.now())+ window_start, window_end = window.start, window.endmax_samples = inputs.max_samples or SAMPLER_MAX_SAMPLES_PER_JOBdiff --git a/posthog/temporal/ai_observability/team_discovery.py b/posthog/temporal/ai_observability/team_discovery.pyindex 9128d4b1f2c7..11bd02a30545 100644--- a/posthog/temporal/ai_observability/team_discovery.py+++ b/posthog/temporal/ai_observability/team_discovery.py@@ -153,6 +153,7 @@ class TeamDiscoveryInput:# a child workflow and its sampling query. Without a window, discovery uses the flag's lookback days.window_start: str | None = Nonewindow_end: str | None = None+ trigger_events: list[str] | None = None@retry(@@ -204,7 +205,9 @@ async def get_team_ids_for_ai_observability(inputs: TeamDiscoveryInput | None =get_teams_with_ai_events,begin,end,- LLM_ANALYTICS_DISCOVERY_TRIGGER_EVENTS,+ inputs.trigger_events+ if inputs is not None and inputs.trigger_events+ else LLM_ANALYTICS_DISCOVERY_TRIGGER_EVENTS,)remaining = [t for t in ai_event_teams if t not in guaranteed and t not in skip]diff --git a/posthog/temporal/ai_observability/test_team_discovery.py b/posthog/temporal/ai_observability/test_team_discovery.pyindex 1eef1e0933cc..ecb9034f63ef 100644--- a/posthog/temporal/ai_observability/test_team_discovery.py+++ b/posthog/temporal/ai_observability/test_team_discovery.py@@ -354,6 +354,14 @@ class TestGetTeamIdsForAIObservability:assert "$ai_generation" in passed_trigger_eventsassert set(passed_trigger_events) < set(AI_OBSERVABILITY_REPORT_TRIGGER_EVENTS)+ @patch("posthog.tasks.ai_observability_usage_report.get_teams_with_ai_events")+ async def test_trigger_events_override_the_default_list(self, mock_get_teams, _mock_ff):+ mock_get_teams.return_value = []++ await get_team_ids_for_ai_observability(TeamDiscoveryInput(trigger_events=["$ai_evaluation"]))++ assert mock_get_teams.call_args.args[2] == ["$ai_evaluation"]+@pytest.mark.parametrize("inputs,expected_window",[