Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): show judge model and backfill context to eval report agent (#121066)

posthog[bot]47b186c34ea1
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
index e97b1b0ab2f2..b1b2c729f0cd 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
@@ -43,8 +43,8 @@ You build the report incrementally by calling three output tools:
## Query tools available
-- **`get_summary_metrics()`**: outcome counts and rates for the current and previous periods. Call this first.
-- **`get_result_distribution_over_time(bucket="hour"|"day")`**: time-series outcome distributions. Use it to spot trends and anomalies.
+- **`get_summary_metrics()`**: outcome counts and rates for the current and previous periods, split by judge model, plus backfills that overlap either period. Call this first.
+- **`get_result_distribution_over_time(bucket="hour"|"day")`**: time-series outcome distributions with runs per judge model. Use it to spot trends and anomalies.
{reasoning_tool_section}- **`list_all_eval_results(max_reasoning_length=80)`**: compact overview of all results, with outcome, target ID, and score when available. {result_overview_detail}
- **`sample_eval_results(outcome="all"|{outcome_options}, limit{sample_ordering_signature})`**: sample evaluation rows. {sample_ordering_instruction}
{detail_tools_section}
@@ -65,6 +65,15 @@ These reports are generated back to back over the same evaluation, so one writte
- Write the TL;DR as a delta: what changed since that report, what held steady, and whether an issue it flagged is now resolved, unchanged, or worse.
- If there are no prior runs, say so and describe this period as the baseline.
+## Comparability rule
+
+A rate can change because the evaluation changed, not because the traffic changed. Check this before you explain a change:
+
+- Read `judge_models` in each period of `get_summary_metrics()`. If the judge model changed between or inside the periods, compare the rates per model. State which model changed and when, from `first_run_at` and `last_run_at`. These are run times, so a backfill that graded past traffic shows its own run time, not the window it graded. Do not attribute the change to traffic, user behavior, or attacks unless the rate also moved under one unchanged model.
+- If a new judge model flags more or fewer results than the old model on comparable traffic, report that as a judge difference and suggest the reader check the new model's results for false positives or false negatives.
+- Read `backfills` in `get_summary_metrics()`. A backfill writes results into past windows after the fact. If a backfill is running and overlaps this period, say that the counts for its window can still grow.
+- A prior report written while a backfill ran saw partial data. If the prior report's `total_runs` is far smaller than the current `previous_period` total for the same window, say that the prior report saw incomplete data and do not repeat its claims about that period.
+
## Suggested workflow
1. Call `get_summary_metrics()`, `list_all_eval_results()`, and `list_recent_report_runs()`.
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py
index 186bd9dd0b7a..761609d65e2f 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools.py
@@ -41,6 +41,7 @@ from posthog.temporal.ai_observability.eval_reports.report_agent.tools import (
_ch_ts,
_dead_backticked_ids,
_execute_ch_query_with_retry,
+ _fetch_overlapping_backfills,
_is_retriable_ch_error,
_label_generation_evals,
_widened_ts_window,
@@ -48,6 +49,7 @@ from posthog.temporal.ai_observability.eval_reports.report_agent.tools import (
add_section,
get_eval_report_tools,
get_report_run,
+ get_result_distribution_over_time,
get_session_detail,
get_summary_metrics,
get_trace_detail,
@@ -71,6 +73,7 @@ _add_citation_fn = add_citation.func # type: ignore[attr-defined]
_list_recent_report_runs_fn = list_recent_report_runs.func # type: ignore[attr-defined]
_get_report_run_fn = get_report_run.func # type: ignore[attr-defined]
_get_summary_metrics_fn = get_summary_metrics.func # type: ignore[attr-defined]
+_get_result_distribution_over_time_fn = get_result_distribution_over_time.func # type: ignore[attr-defined]
_list_all_eval_results_fn = list_all_eval_results.func # type: ignore[attr-defined]
_sample_eval_results_fn = sample_eval_results.func # type: ignore[attr-defined]
_sample_trace_details_fn = sample_trace_details.func # type: ignore[attr-defined]
@@ -173,11 +176,20 @@ class TestWidenedTsWindow(SimpleTestCase):
class TestSummaryMetrics(SimpleTestCase):
+ def setUp(self) -> None:
+ backfills_patcher = patch(
+ "posthog.temporal.ai_observability.eval_reports.report_agent.tools._fetch_overlapping_backfills",
+ return_value=[],
+ )
+ self.mock_backfills = backfills_patcher.start()
+ self.addCleanup(backfills_patcher.stop)
+
@patch("posthog.temporal.ai_observability.eval_reports.report_agent.tools._execute_hogql")
def test_boolean_keeps_pass_rate_separate_from_outcome_distribution(self, mock_execute_hogql):
mock_execute_hogql.side_effect = [
[[80, 18, 2, 100]],
[[7, 2, 1, 10]],
+ [],
]
state = {
"team_id": 1,
@@ -204,6 +216,7 @@ class TestSummaryMetrics(SimpleTestCase):
mock_execute_hogql.side_effect = [
[[0, 0, 4, 4]],
[[0, 0, 3, 3]],
+ [],
]
state = {
"team_id": 1,
@@ -224,6 +237,7 @@ class TestSummaryMetrics(SimpleTestCase):
mock_execute_hogql.side_effect = [
[[2, 1, 1, 4]],
[[1, 1, 0, 2]],
+ [],
]
state = {
"team_id": 1,
@@ -256,6 +270,7 @@ class TestSummaryMetrics(SimpleTestCase):
mock_execute_hogql.side_effect = [
[[18, 80, 2, 100]],
[[2, 7, 1, 10]],
+ [],
]
state = {
"team_id": 1,
@@ -281,6 +296,7 @@ class TestSummaryMetrics(SimpleTestCase):
mock_execute_hogql.side_effect = [
[[80, 18, 2, 100]],
[[7, 2, 1, 10]],
+ [],
]
state = {
"team_id": 1,
@@ -298,6 +314,70 @@ class TestSummaryMetrics(SimpleTestCase):
assert pass_column is not None
self.assertIn("properties.$ai_evaluation_result = 'true'", pass_column.group(1))
+ @patch("posthog.temporal.ai_observability.eval_reports.report_agent.tools._execute_hogql")
+ def test_splits_each_period_by_judge_model_and_reports_overlapping_backfills(self, mock_execute_hogql):
+ mock_execute_hogql.side_effect = [
+ [[0, 0, 0, 300]],
+ [[0, 0, 0, 100]],
+ [
+ ["previous_period", "judge-a", "2026-04-08 13:00:00", "2026-04-08 13:59:00", 99, 1, 0, 100],
+ ["current_period", "judge-a", "2026-04-08 14:00:00", "2026-04-08 14:29:00", 99, 1, 0, 100],
+ ["current_period", "judge-b", "2026-04-08 14:30:00", "2026-04-08 14:59:00", 96, 4, 0, 100],
+ ["current_period", "", "2026-04-08 14:00:00", "2026-04-08 14:59:00", 0, 0, 100, 100],
+ ],
+ ]
+ backfill = {
+ "status": "running",
+ "window_start": "2026-04-01T00:00:00+00:00",
+ "window_end": "2026-04-08T14:00:00+00:00",
+ "created_at": "2026-04-08T14:05:00+00:00",
+ "finished_at": None,
+ }
+ self.mock_backfills.return_value = [backfill]
+ state = {
+ "team_id": 1,
+ "evaluation_id": "eval-id",
+ "output_type": "boolean",
+ "true_is_failure": True,
+ "period_start": "2026-04-08T14:00:00+00:00",
+ "period_end": "2026-04-08T15:00:00+00:00",
+ "previous_period_start": "2026-04-08T13:00:00+00:00",
+ }
+
+ result = json.loads(_get_summary_metrics_fn(state=state))
+
+ current_models = result["current_period"]["judge_models"]
+ self.assertEqual([model["model"] for model in current_models], ["judge-a", "judge-b"])
+ self.assertEqual(current_models[1]["first_run_at"], "2026-04-08 14:30:00")
+ self.assertEqual(current_models[0]["result_rates"]["fail"], 1.0)
+ self.assertEqual(current_models[1]["result_rates"]["fail"], 4.0)
+ self.assertEqual([model["model"] for model in result["previous_period"]["judge_models"]], ["judge-a"])
+ self.assertEqual(result["backfills"], [backfill])
+
+ @patch("posthog.temporal.ai_observability.eval_reports.report_agent.tools._execute_hogql")
+ def test_distribution_reports_runs_per_judge_model_without_unmodeled_results(self, mock_execute_hogql):
+ mock_execute_hogql.return_value = [
+ ["2026-04-08 14:00:00", {"": 5, "judge-a": 10, "judge-b": 10}, 15, 5, 5, 25],
+ ["2026-04-08 15:00:00", {"judge-b": 10}, 7, 3, 0, 10],
+ ["2026-04-08 16:00:00", {"": 4}, 0, 0, 4, 4],
+ ]
+ state = {
+ "team_id": 1,
+ "evaluation_id": "eval-id",
+ "output_type": "boolean",
+ "period_start": "2026-04-08T14:00:00+00:00",
+ "period_end": "2026-04-08T16:00:00+00:00",
+ }
+
+ result = json.loads(_get_result_distribution_over_time_fn(state=state))
+
+ self.assertEqual(len(result), 3)
+ self.assertEqual(result[0]["total_runs"], 25)
+ self.assertEqual(result[0]["result_counts"], {"pass": 15, "fail": 5, "na": 5})
+ self.assertEqual(result[0]["judge_model_runs"], {"judge-a": 10, "judge-b": 10})
+ self.assertEqual(result[1]["judge_model_runs"], {"judge-b": 10})
+ self.assertNotIn("judge_model_runs", result[2])
+
class TestTargetAwareEvalResults(SimpleTestCase):
def _state(self, evaluation_target: str, output_type: str = "boolean") -> dict:
@@ -1363,6 +1443,30 @@ class TestListAndGetReportRun(BaseTest):
result = json.loads(_get_report_run_fn(state=self.state, run_id=str(other_run.id)))
self.assertIn("error", result)
+ def test_fetch_overlapping_backfills_returns_only_windows_that_overlap_the_periods(self):
+ from products.ai_observability.backend.models.evaluation_backfill import EvaluationBackfill
+
+ def create_backfill(window_start: dt.datetime, window_end: dt.datetime) -> None:
+ EvaluationBackfill.objects.for_team(self.team.id).create(
+ team=self.team,
+ evaluation=self.evaluation,
+ window_start=window_start,
+ window_end=window_end,
+ status="completed",
+ target="generation",
+ total_count=10,
+ )
+
+ ts_prev_start = dt.datetime(2026, 4, 1, tzinfo=dt.UTC)
+ ts_end = dt.datetime(2026, 4, 15, tzinfo=dt.UTC)
+ create_backfill(dt.datetime(2026, 3, 25, tzinfo=dt.UTC), dt.datetime(2026, 4, 2, tzinfo=dt.UTC))
+ create_backfill(dt.datetime(2026, 3, 1, tzinfo=dt.UTC), dt.datetime(2026, 4, 1, tzinfo=dt.UTC))
+ create_backfill(dt.datetime(2026, 4, 15, tzinfo=dt.UTC), dt.datetime(2026, 4, 20, tzinfo=dt.UTC))
+
+ backfills = _fetch_overlapping_backfills(self.team.id, str(self.evaluation.id), ts_prev_start, ts_end)
+
+ self.assertEqual([backfill["window_start"] for backfill in backfills], ["2026-03-25T00:00:00+00:00"])
+
class TestExecuteChQueryWithRetry(SimpleTestCase):
@patch("posthog.temporal.ai_observability.eval_reports.report_agent.tools.time.sleep")
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py b/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py
index 2ad393c7000c..7a0044e6469a 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py
@@ -433,6 +433,109 @@ def _period_summary_dict(
return summary
+@frozen
+class JudgeModelSummary:
+ period: Literal["current_period", "previous_period"]
+ model: str
+ first_run_at: str
+ last_run_at: str
+ result_counts: dict[str, int]
+ total_runs: int
+
+
+def _fetch_judge_model_summary(
+ team_id: int,
+ evaluation_id: str,
+ ts_prev_start: datetime,
+ ts_start: datetime,
+ ts_end: datetime,
+ definition: EvaluationReportOutcomeDefinition,
+ evaluation_target: str,
+) -> list[JudgeModelSummary]:
+ """Group outcome counts by judge model for the previous and current periods.
+
+ A judge model switch moves the outcome rates on its own, so the agent needs the
+ split to tell a model change apart from a change in traffic. Results without a
+ model (Hog evaluations, skips that never reached a provider) are left out.
+
+ Run times come from the execution start, not the event timestamp, because a
+ backfilled result carries the timestamp of the generation it grades.
+ """
+ rows = _execute_hogql(
+ team_id,
+ f"""
+ SELECT
+ if(timestamp >= {{ts_start}}, 'current_period', 'previous_period') as period,
+ coalesce(properties.$ai_model, '') as model,
+ min(coalesce(toDateTime(properties.$ai_evaluation_start_time), timestamp)) as first_run_at,
+ max(coalesce(toDateTime(properties.$ai_evaluation_start_time), timestamp)) as last_run_at,
+ {_summary_select_sql(definition)}
+ FROM events
+ WHERE event = '$ai_evaluation'
+ AND properties.$ai_evaluation_id = {{evaluation_id}}
+ AND {definition.event_predicate}
+ AND {target_event_predicate(evaluation_target)}
+ AND timestamp >= {{ts_prev_start}}
+ AND timestamp < {{ts_end}}
+ GROUP BY period, model
+ ORDER BY period, first_run_at, model
+ """,
+ placeholders={
+ "evaluation_id": ast.Constant(value=evaluation_id),
+ "ts_prev_start": ast.Constant(value=ts_prev_start),
+ "ts_start": ast.Constant(value=ts_start),
+ "ts_end": ast.Constant(value=ts_end),
+ },
+ )
+ summaries = []
+ for row in rows:
+ period, model, first_run_at, last_run_at = row[:4]
+ if not model or period not in ("current_period", "previous_period"):
+ continue
+ result_counts, total = _parse_summary_row(definition, row[4:])
+ summaries.append(
+ JudgeModelSummary(
+ period=period,
+ model=str(model),
+ first_run_at=str(first_run_at),
+ last_run_at=str(last_run_at),
+ result_counts=result_counts,
+ total_runs=total,
+ )
+ )
+ return summaries
+
+
+_MAX_REPORTED_BACKFILLS = 5
+
+
+def _fetch_overlapping_backfills(
+ team_id: int, evaluation_id: str, ts_prev_start: datetime, ts_end: datetime
+) -> list[dict[str, str | None]]:
+ """Return backfills whose window overlaps the previous or current period.
+
+ A backfill writes results into past windows after the fact, so a report written
+ while one ran saw partial counts for that window.
+ """
+ from products.ai_observability.backend.models.evaluation_backfill import EvaluationBackfill
+
+ backfills = (
+ EvaluationBackfill.objects.for_team(team_id)
+ .filter(evaluation_id=evaluation_id, window_start__lt=ts_end, window_end__gt=ts_prev_start)
+ .order_by("-created_at")[:_MAX_REPORTED_BACKFILLS]
+ )
+ return [
+ {
+ "status": backfill.status,
+ "window_start": backfill.window_start.isoformat(),
+ "window_end": backfill.window_end.isoformat(),
+ "created_at": backfill.created_at.isoformat(),
+ "finished_at": backfill.finished_at.isoformat() if backfill.finished_at else None,
+ }
+ for backfill in backfills
+ ]
+
+
@tool
def get_summary_metrics(
state: Annotated[dict, InjectedState],
@@ -440,9 +543,12 @@ def get_summary_metrics(
"""Get outcome counts and rates for the current and previous periods.
Returns current and previous period statistics for comparison. Call this first
- to understand the baseline. Note: these numbers are also computed mechanically
- after you finish and attached to the report as `metrics` — you don't need to
- restate them in your sections, just reference them analytically.
+ to understand the baseline. Each period lists `judge_models` with per-model
+ counts, rates, and first/last run times when the evaluation records a judge
+ model. `backfills` lists backfills whose window overlaps either period.
+ Note: these numbers are also computed mechanically after you finish and
+ attached to the report as `metrics` — you don't need to restate them in your
+ sections, just reference them analytically.
"""
team_id = state["team_id"]
evaluation_id = state["evaluation_id"]
@@ -459,16 +565,35 @@ def get_summary_metrics(
team_id, evaluation_id, ts_prev_start, ts_start, definition, evaluation_target
)
- result = {
+ result: dict = {
"output_type": output_type,
"current_period": _period_summary_dict(output_type, result_counts, total),
"previous_period": _period_summary_dict(
output_type, previous_result_counts, previous_total, empty_rates_as_none=True
),
}
+ judge_models = _fetch_judge_model_summary(
+ team_id, evaluation_id, ts_prev_start, ts_start, ts_end, definition, evaluation_target
+ )
+ for model in judge_models:
+ result[model.period].setdefault("judge_models", []).append(
+ {
+ "model": model.model,
+ "first_run_at": model.first_run_at,
+ "last_run_at": model.last_run_at,
+ **_period_summary_dict(output_type, model.result_counts, model.total_runs),
+ }
+ )
+ backfills = _fetch_overlapping_backfills(team_id, evaluation_id, ts_prev_start, ts_end)
+ if backfills:
+ result["backfills"] = backfills
return json.dumps(result, indent=2)
+# A week of hourly buckets already passes HogQL's default limit of 100 rows.
+_MAX_DISTRIBUTION_BUCKETS = 1000
+
+
@tool
def get_result_distribution_over_time(
state: Annotated[dict, InjectedState],
@@ -477,6 +602,8 @@ def get_result_distribution_over_time(
"""Get time-series outcome counts and rates bucketed by hour or day.
Use this to spot trends, anomalies, or degradations over the report period.
+ Each bucket includes `judge_model_runs` (runs per judge model) when the
+ evaluation records a judge model, so a model switch shows up in the series.
Args:
bucket: Time bucket size - "hour" or "day" (default "hour")
@@ -497,6 +624,7 @@ def get_result_distribution_over_time(
f"""
SELECT
{trunc_fn}(timestamp) as bucket,
+ sumMap(map(coalesce(properties.$ai_model, ''), 1)) as judge_model_runs,
{_summary_select_sql(definition)}
FROM events
WHERE event = '$ai_evaluation'
@@ -507,6 +635,7 @@ def get_result_distribution_over_time(
AND timestamp < {{ts_end}}
GROUP BY bucket
ORDER BY bucket
+ LIMIT {_MAX_DISTRIBUTION_BUCKETS}
""",
placeholders={
"evaluation_id": ast.Constant(value=evaluation_id),
@@ -517,8 +646,12 @@ def get_result_distribution_over_time(
result = []
for row in rows:
- result_counts, total = _parse_summary_row(definition, row[1:])
- result.append({"bucket": str(row[0]), **_period_summary_dict(output_type, result_counts, total)})
+ result_counts, total = _parse_summary_row(definition, row[2:])
+ entry = {"bucket": str(row[0]), **_period_summary_dict(output_type, result_counts, total)}
+ judge_model_runs = {str(model): int(runs) for model, runs in (row[1] or {}).items() if model}
+ if judge_model_runs:
+ entry["judge_model_runs"] = judge_model_runs
+ result.append(entry)
return json.dumps(result, indent=2)