Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): discover eval sampler teams from the run's window (#122573)

Bernat Torres46fc6960d679
diff --git a/posthog/temporal/ai_observability/evaluation_clustering/coordinator.py b/posthog/temporal/ai_observability/evaluation_clustering/coordinator.py
index e9a4c431e795..6bb7a150e62e 100644
--- a/posthog/temporal/ai_observability/evaluation_clustering/coordinator.py
+++ b/posthog/temporal/ai_observability/evaluation_clustering/coordinator.py
@@ -26,6 +26,8 @@ from posthog.temporal.ai_observability.evaluation_clustering.models import (
from posthog.temporal.ai_observability.evaluation_clustering.workflow import (
AIObservabilityEvaluationClusteringWorkflow,
AIObservabilityEvaluationSamplerWorkflow,
+ SamplerWindow,
+ sampler_window,
)
from posthog.temporal.ai_observability.trace_clustering.constants import WORKFLOW_EXECUTION_TIMEOUT
from posthog.temporal.common.base import PostHogWorkflow
@@ -70,9 +72,10 @@ class _LevelCoordinatorInputs:
results_so_far: dict[str, Any] | None = None
[email protected](frozen=False)
class SamplerCoordinatorInputs(_LevelCoordinatorInputs):
- pass
+ window_start: str | None = None
+ window_end: str | None = None
@dataclasses.dataclass
@@ -90,12 +93,14 @@ def _empty_results() -> dict[str, Any]:
}
-async def _discover_teams_and_jobs() -> tuple[list[int], dict[int, list[JobConfig]]]:
+async def _discover_teams_and_jobs(
+ discovery: TeamDiscoveryInput | None = None,
+) -> tuple[list[int], dict[int, list[JobConfig]]]:
"""Team discovery + fetch all clustering jobs, reusing the shared LLMA activities."""
try:
team_ids = await workflow.execute_activity(
get_team_ids_for_ai_observability,
- TeamDiscoveryInput(),
+ discovery or TeamDiscoveryInput(),
start_to_close_timeout=DISCOVERY_ACTIVITY_TIMEOUT,
retry_policy=DISCOVERY_ACTIVITY_RETRY_POLICY,
)
@@ -120,6 +125,14 @@ async def _discover_teams_and_jobs() -> tuple[list[int], dict[int, list[JobConfi
return team_ids, per_team_jobs
+def _run_window(inputs: SamplerCoordinatorInputs) -> SamplerWindow:
+ if inputs.window_start and inputs.window_end:
+ return SamplerWindow(start=inputs.window_start, end=inputs.window_end)
+ # workflow_start_time, not now(): a continue_as_new leg carries the window, and a run that a worker picks
+ # up late must still sample the hour it was scheduled for.
+ return sampler_window(workflow.info().workflow_start_time)
+
+
def _evaluation_jobs_for_team(team_jobs: list[JobConfig]) -> list[JobConfig]:
"""Pick the evaluation-level jobs for a team.
@@ -145,6 +158,8 @@ class AIObservabilityEvaluationSamplerCoordinatorWorkflow(PostHogWorkflow):
@workflow.run
async def run(self, inputs: SamplerCoordinatorInputs) -> SamplerCoordinatorResult:
+ window = _run_window(inputs)
+ window_start, window_end = window.start, window.end
if inputs.remaining_team_ids is not None:
team_ids = inputs.remaining_team_ids
per_team_jobs: dict[int, list[JobConfig]] = {}
@@ -154,7 +169,11 @@ class AIObservabilityEvaluationSamplerCoordinatorWorkflow(PostHogWorkflow):
results_so_far = inputs.results_so_far or _empty_results()
else:
logger.info("Starting evaluation sampler coordinator")
- team_ids, per_team_jobs = await _discover_teams_and_jobs()
+ # Most teams with AI events have no $ai_evaluation events in a given hour, and each one would cost a
+ # child workflow and a sampling query that finds nothing. The event and window match the sampler query.
+ team_ids, per_team_jobs = await _discover_teams_and_jobs(
+ TeamDiscoveryInput(window_start=window_start, window_end=window_end, trigger_events=["$ai_evaluation"])
+ )
record_teams_discovered(len(team_ids), "eval_sampling", "evaluation")
results_so_far = _empty_results()
@@ -174,6 +193,8 @@ class AIObservabilityEvaluationSamplerCoordinatorWorkflow(PostHogWorkflow):
job_id=job.job_id,
job_name=job.name,
event_filters=job.event_filters,
+ window_start=window_start,
+ window_end=window_end,
),
id=(f"{SAMPLER_CHILD_WORKFLOW_ID_PREFIX}-{team_id}-{job.job_id}-{workflow.now().isoformat()}"),
execution_timeout=SAMPLER_WORKFLOW_EXECUTION_TIMEOUT,
@@ -207,6 +228,8 @@ class AIObservabilityEvaluationSamplerCoordinatorWorkflow(PostHogWorkflow):
remaining_team_ids=remaining,
per_team_jobs=serializable_jobs,
results_so_far=results_so_far,
+ window_start=window_start,
+ window_end=window_end,
)
)
diff --git a/posthog/temporal/ai_observability/evaluation_clustering/tests/test_workflow.py b/posthog/temporal/ai_observability/evaluation_clustering/tests/test_workflow.py
index f430ffc979a0..a40c6048918a 100644
--- a/posthog/temporal/ai_observability/evaluation_clustering/tests/test_workflow.py
+++ b/posthog/temporal/ai_observability/evaluation_clustering/tests/test_workflow.py
@@ -6,7 +6,8 @@ from datetime import UTC, datetime, timedelta
import pytest
from unittest.mock import patch
-from temporalio import activity
+from parameterized import parameterized
+from temporalio import activity, workflow
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
@@ -22,18 +23,25 @@ from posthog.temporal.ai_observability.evaluation_clustering.constants import (
SAMPLER_MAX_SAMPLES_PER_JOB,
SAMPLER_WINDOW_MINUTES,
SAMPLER_WINDOW_OFFSET_MINUTES,
+ SAMPLER_WORKFLOW_NAME,
+)
+from posthog.temporal.ai_observability.evaluation_clustering.coordinator import (
+ AIObservabilityEvaluationSamplerCoordinatorWorkflow,
+ SamplerCoordinatorInputs,
+ _evaluation_jobs_for_team,
)
-from posthog.temporal.ai_observability.evaluation_clustering.coordinator import _evaluation_jobs_for_team
from posthog.temporal.ai_observability.evaluation_clustering.models import (
SamplerActivityInputs,
SamplerActivityResult,
SamplerWorkflowInputs,
+ SamplerWorkflowResult,
)
from posthog.temporal.ai_observability.evaluation_clustering.workflow import (
AIObservabilityEvaluationClusteringWorkflow,
AIObservabilityEvaluationSamplerWorkflow,
)
-from posthog.temporal.ai_observability.shared_activities import JobConfig
+from posthog.temporal.ai_observability.shared_activities import FetchAllClusteringJobsInput, JobConfig
+from posthog.temporal.ai_observability.team_discovery import TeamDiscoveryInput
from posthog.temporal.ai_observability.trace_clustering.models import ClusteringMetrics, ClusteringResult, ClusterLabel
@@ -188,6 +196,75 @@ class TestSamplerWorkflowWindowMath:
assert activity_inputs.event_filters == filters
+sampler_child_windows: list[tuple[str, str]] = []
+sampler_discovery_inputs: list[TeamDiscoveryInput | None] = []
+
+
[email protected](name=SAMPLER_WORKFLOW_NAME)
+class FakeEvaluationSamplerWorkflow:
+ @workflow.run
+ async def run(self, inputs: SamplerWorkflowInputs) -> SamplerWorkflowResult:
+ if not workflow.unsafe.is_replaying():
+ sampler_child_windows.append((inputs.window_start or "", inputs.window_end or ""))
+ # A later continue_as_new leg then starts minutes after the first, so it would derive a different window.
+ await workflow.sleep(timedelta(minutes=10))
+ return SamplerWorkflowResult(
+ team_id=inputs.team_id,
+ job_id=inputs.job_id,
+ sampled=0,
+ embedded=0,
+ window_start=inputs.window_start or "",
+ window_end=inputs.window_end or "",
+ )
+
+
[email protected](name="get_team_ids_for_llm_analytics")
+async def fake_eval_team_discovery(inputs: TeamDiscoveryInput | None = None) -> list[int]:
+ sampler_discovery_inputs.append(inputs)
+ return [1, 2]
+
+
[email protected](name="fetch_all_clustering_jobs_activity")
+async def fake_fetch_eval_jobs(inputs: FetchAllClusteringJobsInput) -> dict[int, list[JobConfig]]:
+ return {
+ team_id: [JobConfig(job_id=f"j{team_id}", name="evals", analysis_level="evaluation", event_filters=[])]
+ for team_id in inputs.team_ids
+ }
+
+
+class TestSamplerCoordinatorWindow:
+ @parameterized.expand([("single_leg", False), ("continue_as_new_each_batch", True)])
+ @pytest.mark.asyncio
+ async def test_children_sample_the_window_discovery_checked(self, _name, continue_as_new):
+ sampler_child_windows.clear()
+ sampler_discovery_inputs.clear()
+ task_queue = str(uuid.uuid4())
+ with patch.object(workflow.Info, "is_continue_as_new_suggested", return_value=continue_as_new):
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with Worker(
+ env.client,
+ task_queue=task_queue,
+ workflows=[AIObservabilityEvaluationSamplerCoordinatorWorkflow, FakeEvaluationSamplerWorkflow],
+ activities=[fake_eval_team_discovery, fake_fetch_eval_jobs],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ await env.client.execute_workflow(
+ AIObservabilityEvaluationSamplerCoordinatorWorkflow.run,
+ SamplerCoordinatorInputs(max_concurrent_teams=1),
+ id=str(uuid.uuid4()),
+ task_queue=task_queue,
+ )
+
+ assert len(sampler_child_windows) == 2
+ windows = set(sampler_child_windows)
+ assert len(windows) == 1
+ window_start, window_end = next(iter(windows))
+ assert _parse_z(window_end) - _parse_z(window_start) == timedelta(minutes=SAMPLER_WINDOW_MINUTES)
+ assert sampler_discovery_inputs == [
+ TeamDiscoveryInput(window_start=window_start, window_end=window_end, trigger_events=["$ai_evaluation"])
+ ]
+
+
class TestClusteringWorkflowWiring:
@pytest.mark.asyncio
async def test_forwards_workflow_run_identity_to_labeling(self) -> None:
diff --git a/posthog/temporal/ai_observability/evaluation_clustering/workflow.py b/posthog/temporal/ai_observability/evaluation_clustering/workflow.py
index 60af092f01ba..ab619816664b 100644
--- a/posthog/temporal/ai_observability/evaluation_clustering/workflow.py
+++ b/posthog/temporal/ai_observability/evaluation_clustering/workflow.py
@@ -1,11 +1,12 @@
"""Per-job workflows for evaluation clustering (Stage A sampler + Stage B clustering stub)."""
-from datetime import timedelta
+from datetime import datetime, timedelta
import structlog
import temporalio.exceptions
from temporalio import workflow
+from posthog.dataclasses import frozen
from posthog.temporal.ai_observability.evaluation_clustering.constants import (
CLUSTERING_WORKFLOW_NAME,
SAMPLER_ACTIVITY_HEARTBEAT,
@@ -33,6 +34,18 @@ from posthog.temporal.common.base import PostHogWorkflow
logger = structlog.get_logger(__name__)
+@frozen
+class SamplerWindow:
+ start: str
+ end: str
+
+
+def sampler_window(now: datetime) -> SamplerWindow:
+ end = now - timedelta(minutes=SAMPLER_WINDOW_OFFSET_MINUTES)
+ start = end - timedelta(minutes=SAMPLER_WINDOW_MINUTES)
+ return SamplerWindow(start=start.strftime("%Y-%m-%dT%H:%M:%SZ"), end=end.strftime("%Y-%m-%dT%H:%M:%SZ"))
+
+
@workflow.defn(name=SAMPLER_WORKFLOW_NAME)
class AIObservabilityEvaluationSamplerWorkflow(PostHogWorkflow):
"""Hourly per-(team, eval ClusteringJob) sampler.
@@ -55,15 +68,12 @@ class AIObservabilityEvaluationSamplerWorkflow(PostHogWorkflow):
@workflow.run
async def run(self, inputs: SamplerWorkflowInputs) -> SamplerWorkflowResult:
- now = workflow.now()
if inputs.window_start and inputs.window_end:
window_start = inputs.window_start
window_end = inputs.window_end
else:
- window_end_dt = now - timedelta(minutes=SAMPLER_WINDOW_OFFSET_MINUTES)
- window_start_dt = window_end_dt - timedelta(minutes=SAMPLER_WINDOW_MINUTES)
- window_start = window_start_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
- window_end = window_end_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+ window = sampler_window(workflow.now())
+ window_start, window_end = window.start, window.end
max_samples = inputs.max_samples or SAMPLER_MAX_SAMPLES_PER_JOB
diff --git a/posthog/temporal/ai_observability/team_discovery.py b/posthog/temporal/ai_observability/team_discovery.py
index 9128d4b1f2c7..11bd02a30545 100644
--- a/posthog/temporal/ai_observability/team_discovery.py
+++ b/posthog/temporal/ai_observability/team_discovery.py
@@ -153,6 +153,7 @@ class TeamDiscoveryInput:
# a child workflow and its sampling query. Without a window, discovery uses the flag's lookback days.
window_start: str | None = None
window_end: str | None = None
+ trigger_events: list[str] | None = None
@retry(
@@ -204,7 +205,9 @@ async def get_team_ids_for_ai_observability(inputs: TeamDiscoveryInput | None =
get_teams_with_ai_events,
begin,
end,
- LLM_ANALYTICS_DISCOVERY_TRIGGER_EVENTS,
+ inputs.trigger_events
+ if inputs is not None and inputs.trigger_events
+ else LLM_ANALYTICS_DISCOVERY_TRIGGER_EVENTS,
)
remaining = [t for t in ai_event_teams if t not in guaranteed and t not in skip]
diff --git a/posthog/temporal/ai_observability/test_team_discovery.py b/posthog/temporal/ai_observability/test_team_discovery.py
index 1eef1e0933cc..ecb9034f63ef 100644
--- a/posthog/temporal/ai_observability/test_team_discovery.py
+++ b/posthog/temporal/ai_observability/test_team_discovery.py
@@ -354,6 +354,14 @@ class TestGetTeamIdsForAIObservability:
assert "$ai_generation" in passed_trigger_events
assert set(passed_trigger_events) < set(AI_OBSERVABILITY_REPORT_TRIGGER_EVENTS)
+ @patch("posthog.tasks.ai_observability_usage_report.get_teams_with_ai_events")
+ async def test_trigger_events_override_the_default_list(self, mock_get_teams, _mock_ff):
+ mock_get_teams.return_value = []
+
+ await get_team_ids_for_ai_observability(TeamDiscoveryInput(trigger_events=["$ai_evaluation"]))
+
+ assert mock_get_teams.call_args.args[2] == ["$ai_evaluation"]
+
@pytest.mark.parametrize(
"inputs,expected_window",
[