:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(dashboards): add eval for sql tiles that ignore dashboard date range (#119567)
Daniel Parker549e56c918da
diff --git a/products/dashboards/evals/eval_dashboard_sql_date_range.py b/products/dashboards/evals/eval_dashboard_sql_date_range.pynew file mode 100644index 000000000000..bfc961ebb346--- /dev/null+++ b/products/dashboards/evals/eval_dashboard_sql_date_range.py@@ -0,0 +1,53 @@+from __future__ import annotations++from typing import Any++from products.dashboards.backend.models.dashboard import Dashboard+from products.dashboards.evals.scorers import SQLTilesFollowDashboardDateRange+from products.posthog_ai.eval_harness.base import SandboxedPrivateEval+from products.posthog_ai.eval_harness.config import SandboxedEvalCase+from products.posthog_ai.eval_harness.harness.context import EvalContext+from products.tasks.backend.facade.agents import CustomPromptSandboxContext++SIGNUPS_DASHBOARD_NAME = "Signups and uploads (SQL)"+BILLING_DASHBOARD_NAME = "Billing and downloads (SQL)"+++def seed_team(context: CustomPromptSandboxContext) -> dict[str, Any]:+ names = [SIGNUPS_DASHBOARD_NAME, BILLING_DASHBOARD_NAME]+ if Dashboard.objects.filter(team_id=context.team_id, name__in=names, deleted=False).exists():+ raise ValueError("Dashboard SQL date range eval requires unused names")+ return {"team_id": context.team_id}+++async def eval_dashboard_sql_date_range(ctx: EvalContext) -> None:+ await SandboxedPrivateEval(+ experiment_name="sandboxed-dashboards-sql-date-range-cli",+ cases=[+ SandboxedEvalCase(+ name="sql_tiles_without_date_hint",+ prompt=(+ f"Create a dashboard named '{SIGNUPS_DASHBOARD_NAME}' with two SQL insights written in HogQL: "+ "daily `signed_up` event counts, and `uploaded_file` event counts by the `file_type` property."+ ),+ setup=seed_team,+ expected={+ "sql_tiles_follow_dashboard_date_range": {"name": SIGNUPS_DASHBOARD_NAME, "min_sql_tiles": 2}+ },+ ),+ SandboxedEvalCase(+ name="sql_tiles_with_last_30_days",+ prompt=(+ f"Create a dashboard named '{BILLING_DASHBOARD_NAME}' for the last 30 days. "+ "Add two SQL insights written in HogQL: the total `amount_usd` of `paid_bill` events by `plan`, "+ "and daily unique users who triggered `downloaded_file`."+ ),+ setup=seed_team,+ expected={+ "sql_tiles_follow_dashboard_date_range": {"name": BILLING_DASHBOARD_NAME, "min_sql_tiles": 2}+ },+ ),+ ],+ scorers=[SQLTilesFollowDashboardDateRange()],+ ctx=ctx,+ )diff --git a/products/dashboards/evals/scorers.py b/products/dashboards/evals/scorers.pyindex 3de4fe2125fd..e62aa92263ba 100644--- a/products/dashboards/evals/scorers.py+++ b/products/dashboards/evals/scorers.py@@ -3,10 +3,26 @@ from __future__ import annotationsimport asynciofrom typing import Any+from pydantic import ValidationError++from posthog.hogql.errors import ExposedHogQLError++from posthog.errors import ExposedCHQueryError+from posthog.hogql_queries.apply_dashboard_filters import (+ apply_dashboard_filters_to_dict,+ resolve_effective_dashboard_filters,+)+from posthog.hogql_queries.query_runner import get_query_runner+from posthog.models.team import Team+from products.dashboards.backend.models.dashboard import Dashboardfrom products.posthog_ai.eval_harness.scorers.contract import AsyncOnlyScorerMixin, Score, Scorerfrom products.product_analytics.backend.facade.models import Insight+# Both ranges are longer than the 30-day window a prompt can ask for, so a query that hardcodes that+# window returns the same rows for both and fails.+DASHBOARD_DATE_RANGES = ("-60d", "-90d")+class SavedDashboardContents(AsyncOnlyScorerMixin, Scorer):def _name(self) -> str:@@ -62,3 +78,84 @@ class SavedDashboardContents(AsyncOnlyScorerMixin, Scorer):}failed = [name for name, passed in checks.items() if not passed]return Score(name=self._name(), score=float(not failed), metadata={"failed_checks": failed})+++def sql_source(query: Any) -> dict[str, Any] | None:+ if not isinstance(query, dict):+ return None+ if query.get("kind") == "HogQLQuery":+ return query+ source = query.get("source")+ if isinstance(source, dict) and source.get("kind") == "HogQLQuery":+ return source+ return None+++class SQLTilesFollowDashboardDateRange(AsyncOnlyScorerMixin, Scorer):+ def _name(self) -> str:+ return "sql_tiles_follow_dashboard_date_range"++ @staticmethod+ def _results_for_date_range(+ source: dict[str, Any], date_from: str, tile_filters_override: dict[str, Any] | None, team: Team+ ) -> str:+ effective = resolve_effective_dashboard_filters(source, {"date_from": date_from}, tile_filters_override)+ query = apply_dashboard_filters_to_dict(effective.query, effective.filters, team)+ # Grouped SQL without ORDER BY can return the same rows in a different order.+ return repr(sorted(repr(row) for row in get_query_runner(query, team).calculate().results))++ @classmethod+ def _read_state(cls, team_id: int, name: str) -> dict[str, Any]:+ team = Team.objects.get(id=team_id)+ matching = list(Dashboard.objects.filter(team_id=team_id, name=name, deleted=False))+ sql_tiles: list[dict[str, Any]] = []+ if len(matching) == 1:+ tiles = matching[0].tiles.filter(insight__isnull=False, insight__deleted=False).select_related("insight")+ for tile in tiles:+ source = sql_source(tile.insight.query)+ if source is None:+ continue+ state: dict[str, Any] = {+ "insight_id": tile.insight_id,+ "uses_filters_placeholder": "{filters" in str(source.get("query", "")),+ }+ try:+ state["results"] = [+ cls._results_for_date_range(source, date_from, tile.filters_overrides, team)+ for date_from in DASHBOARD_DATE_RANGES+ ]+ # Other exceptions are infrastructure failures, so they reach the harness as scorer errors.+ except (ExposedHogQLError, ExposedCHQueryError, ValidationError) as error:+ state["error"] = repr(error)+ sql_tiles.append(state)+ return {"matching_count": len(matching), "sql_tiles": sql_tiles}++ async def _run_eval_async(+ self, output: dict[str, Any] | None, expected: dict[str, Any] | None = None, **kwargs: Any+ ) -> Score:+ seed = (output or {}).get("seed") or {}+ spec = (expected or {}).get(self._name())+ if not seed.get("team_id") or not spec:+ return Score(name=self._name(), score=0.0, metadata={"reason": "Missing seed or expected dashboard"})++ state = await asyncio.to_thread(self._read_state, seed["team_id"], spec["name"])+ if state["matching_count"] != 1:+ return Score(+ name=self._name(), score=0.0, metadata={"reason": "Expected one dashboard with the requested name"}+ )++ tiles = state["sql_tiles"]+ follows = {str(tile["insight_id"]): "error" not in tile and len(set(tile["results"])) > 1 for tile in tiles}+ required = max(len(tiles), spec.get("min_sql_tiles", 1))+ return Score(+ name=self._name(),+ score=sum(follows.values()) / required,+ metadata={+ "sql_tile_count": len(tiles),+ "follows_date_range": follows,+ "uses_filters_placeholder": {+ str(tile["insight_id"]): tile["uses_filters_placeholder"] for tile in tiles+ },+ "errors": {str(tile["insight_id"]): tile["error"] for tile in tiles if "error" in tile},+ },+ )diff --git a/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py b/products/posthog_ai/eval_harness/test/test_saved_object_scorers.pyindex ad4efc0cc9c7..297d40f2e43f 100644--- a/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py+++ b/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py@@ -2,15 +2,22 @@ from __future__ import annotationsimport asynciofrom copy import deepcopy+from datetime import timedeltafrom typing import Any+from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_eventsfrom unittest.mock import patch+from django.utils import timezone+from parameterized import parameterized-from products.dashboards.evals.scorers import SavedDashboardContents+from products.dashboards.backend.models.dashboard import Dashboard+from products.dashboards.backend.models.dashboard_tile import DashboardTile+from products.dashboards.evals.scorers import SavedDashboardContents, SQLTilesFollowDashboardDateRangefrom products.feature_flags.backend.models.feature_flag import FeatureFlagfrom products.feature_flags.evals.creation_scorers import CreatedFlagConfiguration+from products.product_analytics.backend.facade.models import Insight@parameterized.expand(@@ -117,3 +124,44 @@ def test_dashboard_scores_saved_charts_and_preserves_existing_tile())assert score.score == expected_score+++class TestSQLTilesFollowDashboardDateRange(ClickhouseTestMixin, APIBaseTest):+ @parameterized.expand(+ [+ ("SELECT count() FROM events WHERE {filters}", None, 1.0),+ ("SELECT count() FROM events WHERE {filters}", {"date_from": "-30d"}, 0.0),+ ("SELECT count() FROM events WHERE {filters} AND timestamp >= now() - INTERVAL 30 DAY", None, 0.0),+ ("SELECT count() FROM events", None, 0.0),+ ("SELECT number FROM numbers(20) ORDER BY rand()", None, 0.0),+ ("SELECT missing_column FROM events WHERE {filters}", None, 0.0),+ ]+ )+ def test_scores_sql_tiles_by_dashboard_date_range_response(+ self, sql: str, filters_overrides: dict[str, Any] | None, expected_score: float+ ) -> None:+ for days_ago in (10, 75):+ _create_event(+ team=self.team,+ event="signed_up",+ distinct_id="user",+ timestamp=timezone.now() - timedelta(days=days_ago),+ )+ flush_persons_and_events()+ dashboard = Dashboard.objects.create(team=self.team, name="SQL dashboard")+ insight = Insight.objects.create(+ team=self.team,+ query={"kind": "DataVisualizationNode", "source": {"kind": "HogQLQuery", "query": sql}},+ )+ DashboardTile.objects.create(dashboard=dashboard, insight=insight, filters_overrides=filters_overrides)++ state = SQLTilesFollowDashboardDateRange._read_state(self.team.id, "SQL dashboard")+ with patch.object(SQLTilesFollowDashboardDateRange, "_read_state", return_value=state):+ score = asyncio.run(+ SQLTilesFollowDashboardDateRange().eval_async(+ {"seed": {"team_id": self.team.id}},+ {"sql_tiles_follow_dashboard_date_range": {"name": "SQL dashboard", "min_sql_tiles": 1}},+ )+ )++ assert score.score == expected_score, score.metadata