Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(dashboards): add eval for sql tiles that ignore dashboard date range (#119567)

Daniel Parker549e56c918da
diff --git a/products/dashboards/evals/eval_dashboard_sql_date_range.py b/products/dashboards/evals/eval_dashboard_sql_date_range.py
new file mode 100644
index 000000000000..bfc961ebb346
--- /dev/null
+++ b/products/dashboards/evals/eval_dashboard_sql_date_range.py
@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from typing import Any
+
+from products.dashboards.backend.models.dashboard import Dashboard
+from products.dashboards.evals.scorers import SQLTilesFollowDashboardDateRange
+from products.posthog_ai.eval_harness.base import SandboxedPrivateEval
+from products.posthog_ai.eval_harness.config import SandboxedEvalCase
+from products.posthog_ai.eval_harness.harness.context import EvalContext
+from products.tasks.backend.facade.agents import CustomPromptSandboxContext
+
+SIGNUPS_DASHBOARD_NAME = "Signups and uploads (SQL)"
+BILLING_DASHBOARD_NAME = "Billing and downloads (SQL)"
+
+
+def seed_team(context: CustomPromptSandboxContext) -> dict[str, Any]:
+ names = [SIGNUPS_DASHBOARD_NAME, BILLING_DASHBOARD_NAME]
+ if Dashboard.objects.filter(team_id=context.team_id, name__in=names, deleted=False).exists():
+ raise ValueError("Dashboard SQL date range eval requires unused names")
+ return {"team_id": context.team_id}
+
+
+async def eval_dashboard_sql_date_range(ctx: EvalContext) -> None:
+ await SandboxedPrivateEval(
+ experiment_name="sandboxed-dashboards-sql-date-range-cli",
+ cases=[
+ SandboxedEvalCase(
+ name="sql_tiles_without_date_hint",
+ prompt=(
+ f"Create a dashboard named '{SIGNUPS_DASHBOARD_NAME}' with two SQL insights written in HogQL: "
+ "daily `signed_up` event counts, and `uploaded_file` event counts by the `file_type` property."
+ ),
+ setup=seed_team,
+ expected={
+ "sql_tiles_follow_dashboard_date_range": {"name": SIGNUPS_DASHBOARD_NAME, "min_sql_tiles": 2}
+ },
+ ),
+ SandboxedEvalCase(
+ name="sql_tiles_with_last_30_days",
+ prompt=(
+ f"Create a dashboard named '{BILLING_DASHBOARD_NAME}' for the last 30 days. "
+ "Add two SQL insights written in HogQL: the total `amount_usd` of `paid_bill` events by `plan`, "
+ "and daily unique users who triggered `downloaded_file`."
+ ),
+ setup=seed_team,
+ expected={
+ "sql_tiles_follow_dashboard_date_range": {"name": BILLING_DASHBOARD_NAME, "min_sql_tiles": 2}
+ },
+ ),
+ ],
+ scorers=[SQLTilesFollowDashboardDateRange()],
+ ctx=ctx,
+ )
diff --git a/products/dashboards/evals/scorers.py b/products/dashboards/evals/scorers.py
index 3de4fe2125fd..e62aa92263ba 100644
--- a/products/dashboards/evals/scorers.py
+++ b/products/dashboards/evals/scorers.py
@@ -3,10 +3,26 @@ from __future__ import annotations
import asyncio
from typing import Any
+from pydantic import ValidationError
+
+from posthog.hogql.errors import ExposedHogQLError
+
+from posthog.errors import ExposedCHQueryError
+from posthog.hogql_queries.apply_dashboard_filters import (
+ apply_dashboard_filters_to_dict,
+ resolve_effective_dashboard_filters,
+)
+from posthog.hogql_queries.query_runner import get_query_runner
+from posthog.models.team import Team
+
from products.dashboards.backend.models.dashboard import Dashboard
from products.posthog_ai.eval_harness.scorers.contract import AsyncOnlyScorerMixin, Score, Scorer
from products.product_analytics.backend.facade.models import Insight
+# Both ranges are longer than the 30-day window a prompt can ask for, so a query that hardcodes that
+# window returns the same rows for both and fails.
+DASHBOARD_DATE_RANGES = ("-60d", "-90d")
+
class SavedDashboardContents(AsyncOnlyScorerMixin, Scorer):
def _name(self) -> str:
@@ -62,3 +78,84 @@ class SavedDashboardContents(AsyncOnlyScorerMixin, Scorer):
}
failed = [name for name, passed in checks.items() if not passed]
return Score(name=self._name(), score=float(not failed), metadata={"failed_checks": failed})
+
+
+def sql_source(query: Any) -> dict[str, Any] | None:
+ if not isinstance(query, dict):
+ return None
+ if query.get("kind") == "HogQLQuery":
+ return query
+ source = query.get("source")
+ if isinstance(source, dict) and source.get("kind") == "HogQLQuery":
+ return source
+ return None
+
+
+class SQLTilesFollowDashboardDateRange(AsyncOnlyScorerMixin, Scorer):
+ def _name(self) -> str:
+ return "sql_tiles_follow_dashboard_date_range"
+
+ @staticmethod
+ def _results_for_date_range(
+ source: dict[str, Any], date_from: str, tile_filters_override: dict[str, Any] | None, team: Team
+ ) -> str:
+ effective = resolve_effective_dashboard_filters(source, {"date_from": date_from}, tile_filters_override)
+ query = apply_dashboard_filters_to_dict(effective.query, effective.filters, team)
+ # Grouped SQL without ORDER BY can return the same rows in a different order.
+ return repr(sorted(repr(row) for row in get_query_runner(query, team).calculate().results))
+
+ @classmethod
+ def _read_state(cls, team_id: int, name: str) -> dict[str, Any]:
+ team = Team.objects.get(id=team_id)
+ matching = list(Dashboard.objects.filter(team_id=team_id, name=name, deleted=False))
+ sql_tiles: list[dict[str, Any]] = []
+ if len(matching) == 1:
+ tiles = matching[0].tiles.filter(insight__isnull=False, insight__deleted=False).select_related("insight")
+ for tile in tiles:
+ source = sql_source(tile.insight.query)
+ if source is None:
+ continue
+ state: dict[str, Any] = {
+ "insight_id": tile.insight_id,
+ "uses_filters_placeholder": "{filters" in str(source.get("query", "")),
+ }
+ try:
+ state["results"] = [
+ cls._results_for_date_range(source, date_from, tile.filters_overrides, team)
+ for date_from in DASHBOARD_DATE_RANGES
+ ]
+ # Other exceptions are infrastructure failures, so they reach the harness as scorer errors.
+ except (ExposedHogQLError, ExposedCHQueryError, ValidationError) as error:
+ state["error"] = repr(error)
+ sql_tiles.append(state)
+ return {"matching_count": len(matching), "sql_tiles": sql_tiles}
+
+ async def _run_eval_async(
+ self, output: dict[str, Any] | None, expected: dict[str, Any] | None = None, **kwargs: Any
+ ) -> Score:
+ seed = (output or {}).get("seed") or {}
+ spec = (expected or {}).get(self._name())
+ if not seed.get("team_id") or not spec:
+ return Score(name=self._name(), score=0.0, metadata={"reason": "Missing seed or expected dashboard"})
+
+ state = await asyncio.to_thread(self._read_state, seed["team_id"], spec["name"])
+ if state["matching_count"] != 1:
+ return Score(
+ name=self._name(), score=0.0, metadata={"reason": "Expected one dashboard with the requested name"}
+ )
+
+ tiles = state["sql_tiles"]
+ follows = {str(tile["insight_id"]): "error" not in tile and len(set(tile["results"])) > 1 for tile in tiles}
+ required = max(len(tiles), spec.get("min_sql_tiles", 1))
+ return Score(
+ name=self._name(),
+ score=sum(follows.values()) / required,
+ metadata={
+ "sql_tile_count": len(tiles),
+ "follows_date_range": follows,
+ "uses_filters_placeholder": {
+ str(tile["insight_id"]): tile["uses_filters_placeholder"] for tile in tiles
+ },
+ "errors": {str(tile["insight_id"]): tile["error"] for tile in tiles if "error" in tile},
+ },
+ )
diff --git a/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py b/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py
index ad4efc0cc9c7..297d40f2e43f 100644
--- a/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py
+++ b/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py
@@ -2,15 +2,22 @@ from __future__ import annotations
import asyncio
from copy import deepcopy
+from datetime import timedelta
from typing import Any
+from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
from unittest.mock import patch
+from django.utils import timezone
+
from parameterized import parameterized
-from products.dashboards.evals.scorers import SavedDashboardContents
+from products.dashboards.backend.models.dashboard import Dashboard
+from products.dashboards.backend.models.dashboard_tile import DashboardTile
+from products.dashboards.evals.scorers import SavedDashboardContents, SQLTilesFollowDashboardDateRange
from products.feature_flags.backend.models.feature_flag import FeatureFlag
from products.feature_flags.evals.creation_scorers import CreatedFlagConfiguration
+from products.product_analytics.backend.facade.models import Insight
@parameterized.expand(
@@ -117,3 +124,44 @@ def test_dashboard_scores_saved_charts_and_preserves_existing_tile(
)
)
assert score.score == expected_score
+
+
+class TestSQLTilesFollowDashboardDateRange(ClickhouseTestMixin, APIBaseTest):
+ @parameterized.expand(
+ [
+ ("SELECT count() FROM events WHERE {filters}", None, 1.0),
+ ("SELECT count() FROM events WHERE {filters}", {"date_from": "-30d"}, 0.0),
+ ("SELECT count() FROM events WHERE {filters} AND timestamp >= now() - INTERVAL 30 DAY", None, 0.0),
+ ("SELECT count() FROM events", None, 0.0),
+ ("SELECT number FROM numbers(20) ORDER BY rand()", None, 0.0),
+ ("SELECT missing_column FROM events WHERE {filters}", None, 0.0),
+ ]
+ )
+ def test_scores_sql_tiles_by_dashboard_date_range_response(
+ self, sql: str, filters_overrides: dict[str, Any] | None, expected_score: float
+ ) -> None:
+ for days_ago in (10, 75):
+ _create_event(
+ team=self.team,
+ event="signed_up",
+ distinct_id="user",
+ timestamp=timezone.now() - timedelta(days=days_ago),
+ )
+ flush_persons_and_events()
+ dashboard = Dashboard.objects.create(team=self.team, name="SQL dashboard")
+ insight = Insight.objects.create(
+ team=self.team,
+ query={"kind": "DataVisualizationNode", "source": {"kind": "HogQLQuery", "query": sql}},
+ )
+ DashboardTile.objects.create(dashboard=dashboard, insight=insight, filters_overrides=filters_overrides)
+
+ state = SQLTilesFollowDashboardDateRange._read_state(self.team.id, "SQL dashboard")
+ with patch.object(SQLTilesFollowDashboardDateRange, "_read_state", return_value=state):
+ score = asyncio.run(
+ SQLTilesFollowDashboardDateRange().eval_async(
+ {"seed": {"team_id": self.team.id}},
+ {"sql_tiles_follow_dashboard_date_range": {"name": "SQL dashboard", "min_sql_tiles": 1}},
+ )
+ )
+
+ assert score.score == expected_score, score.metadata