Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(mcp): make agents save sql insights that follow the date range (#123974)

Daniel Parkerdbd040dc2480
diff --git a/products/posthog_ai/eval_harness/test/test_sql_insight_scorers.py b/products/posthog_ai/eval_harness/test/test_sql_insight_scorers.py
new file mode 100644
index 000000000000..404b70ac135d
--- /dev/null
+++ b/products/posthog_ai/eval_harness/test/test_sql_insight_scorers.py
@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import pytest
+
+from products.posthog_ai.eval_harness.test.test_eval_scorers import _raw_tool_log
+from products.posthog_ai.evals.product_analytics.sql_insight_scorers import (
+ SavedSqlInsight,
+ SqlFollowsDateFilter,
+ SqlTimeAxisAscending,
+)
+
+DEFAULT_RANGE = {"dateRange": {"date_from": "-14d"}}
+
+
+def _save(
+ sql: str, filters: dict | None = DEFAULT_RANGE, dashboards: list[int] | None = None, result: str = '{"id": 1}'
+) -> tuple:
+ source: dict = {"kind": "HogQLQuery", "query": sql}
+ if filters is not None:
+ source["filters"] = filters
+ payload: dict = {"name": "Uploads", "query": {"kind": "DataVisualizationNode", "source": source}}
+ if dashboards is not None:
+ payload["dashboards"] = dashboards
+ return ("mcp__posthog__insight-create", payload, result)
+
+
+def _add_to_dashboard(insight_id: int | str) -> tuple:
+ return ("mcp__posthog__insight-update", {"id": insight_id, "dashboards": [7]}, '{"id": 1}')
+
+
+def _output(*calls: tuple) -> dict:
+ return {"raw_log": _raw_tool_log(list(calls)), "prompt": "Save a SQL insight"}
+
+
+ "sql,filters,expected_score",
+ [
+ ("SELECT count() FROM events WHERE {filters}", DEFAULT_RANGE, 1.0),
+ (
+ "WITH firsts AS (SELECT person_id, min(timestamp) AS t FROM events WHERE {filters} GROUP BY person_id) "
+ "SELECT count() FROM firsts",
+ DEFAULT_RANGE,
+ 1.0,
+ ),
+ (
+ "SELECT count() FROM events WHERE timestamp >= {filters.dateRange.from} "
+ "AND timestamp < {filters.dateRange.to}",
+ DEFAULT_RANGE,
+ 1.0,
+ ),
+ ("SELECT count() FROM events WHERE timestamp >= now() - INTERVAL 14 DAY", DEFAULT_RANGE, 0.0),
+ ("SELECT count() FROM events WHERE {filters} AND timestamp >= now() - INTERVAL 14 DAY", DEFAULT_RANGE, 0.0),
+ ("SELECT count() FROM events WHERE timestamp BETWEEN '2026-01-01' AND '2026-01-15'", DEFAULT_RANGE, 0.0),
+ ("SELECT count() FROM events WHERE {filters}", None, 0.0),
+ ("SELECT count() FROM events WHERE {filters(created_at AS timestamp)}", DEFAULT_RANGE, 1.0),
+ ("SELECT count() FROM events WHERE {filters(null AS timestamp)}", DEFAULT_RANGE, 0.0),
+ (
+ "SELECT dateTrunc({filters.interval}, timestamp) AS bucket, count() FROM events GROUP BY bucket",
+ DEFAULT_RANGE,
+ 0.0,
+ ),
+ (
+ "SELECT count() FROM events WHERE {filters} AND timestamp >= toDateTime('2026-01-01 00:00:00')",
+ DEFAULT_RANGE,
+ 0.0,
+ ),
+ ("SELECT count() FROM events WHERE timestamp BETWEEN {filters.dateRange.from} AND now()", DEFAULT_RANGE, 0.0),
+ ],
+ ids=[
+ "filters_placeholder",
+ "filters_inside_cte",
+ "date_range_placeholders",
+ "hard_coded_window",
+ "placeholder_plus_hard_coded_window",
+ "date_literal_window",
+ "no_default_date_range",
+ "column_bound_timestamp",
+ "column_bound_null_timestamp",
+ "interval_placeholder_only",
+ "date_literal_inside_conversion",
+ "placeholder_and_clock_in_one_window",
+ ],
+)
+def test_sql_follows_date_filter(sql: str, filters: dict | None, expected_score: float) -> None:
+ score = SqlFollowsDateFilter()._run_eval_sync(
+ _output(_save(sql, filters)), expected={"sql_follows_date_filter": {}}
+ )
+
+ assert score.score == expected_score, score.metadata
+
+
+DAILY_BY_FILE_TYPE = (
+ "SELECT toStartOfDay(timestamp) AS day, properties.file_type AS file_type, count() AS uploads "
+ "FROM events WHERE {filters} GROUP BY day, file_type "
+)
+DAY_SERIES_JOIN = (
+ "SELECT d.day AS day, coalesce(c.uploads, 0) AS uploads "
+ "FROM (SELECT today() - number AS day FROM numbers(30)) AS d "
+ "LEFT JOIN (SELECT toDate(timestamp) AS day, count() AS uploads FROM events WHERE {filters} GROUP BY day) AS c "
+ "ON d.day = c.day "
+)
+
+
+ "sql,expected_score",
+ [
+ (DAILY_BY_FILE_TYPE + "ORDER BY day", 1.0),
+ (DAILY_BY_FILE_TYPE + "ORDER BY day ASC", 1.0),
+ (DAILY_BY_FILE_TYPE + "ORDER BY 1", 1.0),
+ (DAILY_BY_FILE_TYPE + "ORDER BY file_type, toStartOfDay(timestamp)", 1.0),
+ (DAY_SERIES_JOIN + "ORDER BY day", 1.0),
+ (
+ "SELECT bucket, sum(uploads) AS uploads FROM (SELECT toStartOfWeek(timestamp) AS bucket, count() AS uploads "
+ "FROM events WHERE {filters} GROUP BY bucket) GROUP BY bucket ORDER BY bucket",
+ 1.0,
+ ),
+ (
+ "WITH weeks AS (SELECT toStartOfWeek(now()) - toIntervalWeek(number) AS week FROM numbers(8)) "
+ "SELECT weeks.week AS week FROM weeks ORDER BY week",
+ 1.0,
+ ),
+ (
+ "WITH daily AS (SELECT toStartOfDay(timestamp) AS day, count() AS uploads FROM events WHERE {filters} "
+ "GROUP BY day) SELECT day, uploads FROM daily ORDER BY day",
+ 1.0,
+ ),
+ ("WITH events AS (SELECT timestamp FROM events) SELECT timestamp FROM events ORDER BY timestamp", 1.0),
+ (
+ DAILY_BY_FILE_TYPE
+ + "ORDER BY day WITH FILL FROM toStartOfDay({filters.dateRange.from}) TO {filters.dateRange.to} "
+ + "STEP toIntervalDay(1)",
+ 1.0,
+ ),
+ (DAILY_BY_FILE_TYPE + "ORDER BY day DESC", 0.0),
+ (DAY_SERIES_JOIN + "ORDER BY day DESC", 0.0),
+ (DAILY_BY_FILE_TYPE + "ORDER BY uploads DESC", 0.0),
+ (DAILY_BY_FILE_TYPE + "ORDER BY uploads DESC, day ASC", 0.0),
+ (DAILY_BY_FILE_TYPE, 0.0),
+ ],
+ ids=[
+ "alias",
+ "alias_asc",
+ "position",
+ "expression_after_breakdown",
+ "generated_day_series_join",
+ "outer_query_over_subquery",
+ "generated_week_series_cte",
+ "bucket_projected_through_cte",
+ "cte_shadows_events_table",
+ "with_fill_bounded_by_placeholders",
+ "desc",
+ "generated_day_series_join_desc",
+ "no_time_key",
+ "value_sorted_before_time",
+ "no_order_by",
+ ],
+)
+def test_sql_time_axis_ascending(sql: str, expected_score: float) -> None:
+ score = SqlTimeAxisAscending()._run_eval_sync(_output(_save(sql)), expected={"sql_time_axis_ascending": {}})
+
+ assert score.score == expected_score, score.metadata
+
+
+ "calls,expected_score",
+ [
+ ([_save("SELECT 1", dashboards=[7])], 1.0),
+ ([_save("SELECT 1"), _add_to_dashboard(1)], 1.0),
+ (
+ [_save("SELECT 1", result="id: 10975\nshort_id: Ab3xYz9q\nname: Uploads"), _add_to_dashboard("Ab3xYz9q")],
+ 1.0,
+ ),
+ ([_save("SELECT 1"), _add_to_dashboard(2)], 0.0),
+ ([_save("SELECT 1")], 0.0),
+ (
+ [
+ (
+ "mcp__posthog__insight-create",
+ {
+ "name": "Uploads",
+ "dashboards": [7],
+ "query": {"kind": "InsightVizNode", "source": {"kind": "TrendsQuery", "series": []}},
+ },
+ '{"id": 1}',
+ )
+ ],
+ 0.0,
+ ),
+ ],
+ ids=[
+ "created_on_dashboard",
+ "added_by_update",
+ "added_by_update_with_text_output_short_id",
+ "other_insight_added_to_dashboard",
+ "not_on_dashboard",
+ "typed_insight_only",
+ ],
+)
+def test_saved_sql_insight_on_dashboard(calls: list[tuple], expected_score: float) -> None:
+ score = SavedSqlInsight()._run_eval_sync(_output(*calls), expected={"saved_sql_insight": {"on_dashboard": True}})
+
+ assert score.score == expected_score, score.metadata
diff --git a/products/posthog_ai/evals/offline_config.py b/products/posthog_ai/evals/offline_config.py
index df6eb0c7a211..efad9a54bd2d 100644
--- a/products/posthog_ai/evals/offline_config.py
+++ b/products/posthog_ai/evals/offline_config.py
@@ -164,7 +164,10 @@ _SCORERS: dict[str, dict[str, OfflineScorer]] = {
kind="boolean", version=UUID("01a15581-6891-7f9b-bbb4-199b8b2edec0")
),
"saved_insight": OfflineScorer(kind="boolean", version=UUID("01a15581-ee89-7754-b0dd-6012efeaa1c4")),
+ "saved_sql_insight": OfflineScorer(kind="boolean", version=UUID("01a17d33-592a-7516-b59c-ab51d460d9de")),
"schema_discovery_order": OfflineScorer(kind="boolean", version=UUID("01a15581-7429-7fff-85a7-9c223feddee6")),
+ "sql_follows_date_filter": OfflineScorer(kind="boolean", version=UUID("01a17d33-6182-7c29-aa06-d8e641975951")),
+ "sql_time_axis_ascending": OfflineScorer(kind="boolean", version=UUID("01a17d33-6cdf-784d-b73b-9ca1ed42971e")),
"trends_schema_alignment": OfflineScorer(kind="numeric", version=UUID("01a15581-a78c-7d03-bb2d-de43ab247f81")),
"trends_time_range_relevancy": OfflineScorer(
kind="boolean", version=UUID("01a15581-b135-7a70-ae09-b3d0e93bc894")
diff --git a/products/posthog_ai/evals/product_analytics/eval_sql_insight_hygiene.py b/products/posthog_ai/evals/product_analytics/eval_sql_insight_hygiene.py
new file mode 100644
index 000000000000..be69b9c712ce
--- /dev/null
+++ b/products/posthog_ai/evals/product_analytics/eval_sql_insight_hygiene.py
@@ -0,0 +1,89 @@
+"""Do the SQL insights the agent saves behave like insights on a dashboard?
+
+Agents save SQL insights with the window from the request written into the query, for example
+``timestamp >= now() - INTERVAL 14 DAY``. The insight then ignores the dashboard and insight
+date picker: the user changes the range and nothing changes. Agents also order a SQL time series
+newest first, which puts the most recent bucket on the left of the chart.
+
+Each case asks for a SQL insight, so SQL is the right tool and the suite grades only how the
+query is written. ``saved_sql_insight`` is the manipulation check: drop a case that saved no
+SQL insight before reading the other scores. ``sql_follows_date_filter`` and
+``sql_time_axis_ascending`` grade the saved query.
+
+Run on the latest Claude Opus and GPT Sol models, one runtime per run:
+
+ hogli evals eval_sql_insight_hygiene --agent-runtime claude --agent-model claude-opus-5-5 --trials 3
+ hogli evals eval_sql_insight_hygiene --agent-runtime codex --agent-model gpt-6.1-sol --trials 3
+"""
+
+from __future__ import annotations
+
+from products.posthog_ai.eval_harness.base import SandboxedEval
+from products.posthog_ai.eval_harness.config import SandboxedEvalCase
+from products.posthog_ai.eval_harness.harness.context import EvalContext
+from products.posthog_ai.evals.offline_config import offline_suite
+from products.posthog_ai.evals.product_analytics.sql_insight_scorers import (
+ SavedSqlInsight,
+ SqlFollowsDateFilter,
+ SqlTimeAxisAscending,
+)
+
+
+def _case(name: str, prompt: str, *, on_dashboard: bool, time_series: bool) -> SandboxedEvalCase:
+ expected: dict[str, dict] = {
+ "saved_sql_insight": {"on_dashboard": on_dashboard},
+ "sql_follows_date_filter": {},
+ }
+ if time_series:
+ expected["sql_time_axis_ascending"] = {}
+ return SandboxedEvalCase(name=name, prompt=prompt, expected=expected)
+
+
+async def eval_sql_insight_hygiene(ctx: EvalContext) -> None:
+ cases = [
+ _case(
+ "daily_uploads_on_dashboard",
+ "Write a HogQL query that counts uploaded_file events per day for the last 14 days, and add it as a "
+ "line chart to the 🔑 Key metrics dashboard.",
+ on_dashboard=True,
+ time_series=True,
+ ),
+ _case(
+ "weekly_median_file_size_on_dashboard",
+ "Add a SQL insight to the 🔑 Key metrics dashboard that shows the median file_size_b of uploaded_file "
+ "events per week over the last 8 weeks.",
+ on_dashboard=True,
+ time_series=True,
+ ),
+ _case(
+ "weekly_revenue_on_dashboard",
+ "Add a SQL insight to the 💸 Revenue dashboard with weekly revenue from paid_bill events, as the sum of "
+ "amount_usd, over the last 12 weeks.",
+ on_dashboard=True,
+ time_series=True,
+ ),
+ _case(
+ "upload_gap_leaderboard_on_dashboard",
+ "Add a SQL insight to the 🔑 Key metrics dashboard that lists the 10 users with the longest gap in hours "
+ "between their first and second uploaded_file event in the last 30 days.",
+ on_dashboard=True,
+ time_series=False,
+ ),
+ _case(
+ "daily_downloading_users_insight",
+ "Save a HogQL insight that charts daily unique users who downloaded_file over the last 30 days.",
+ on_dashboard=False,
+ time_series=True,
+ ),
+ ]
+
+ await SandboxedEval(
+ experiment_name="sandboxed-sql-insight-hygiene-cli",
+ offline_suite=offline_suite(
+ "product_analytics/eval_sql_insight_hygiene::eval_sql_insight_hygiene",
+ metrics=["saved_sql_insight", "sql_follows_date_filter", "sql_time_axis_ascending"],
+ ),
+ cases=cases,
+ scorers=[SavedSqlInsight(), SqlFollowsDateFilter(), SqlTimeAxisAscending()],
+ ctx=ctx,
+ )
diff --git a/products/posthog_ai/evals/product_analytics/sql_insight_scorers.py b/products/posthog_ai/evals/product_analytics/sql_insight_scorers.py
new file mode 100644
index 000000000000..fdeefc5b3d09
--- /dev/null
+++ b/products/posthog_ai/evals/product_analytics/sql_insight_scorers.py
@@ -0,0 +1,423 @@
+"""Scorers for the SQL insights an agent saves.
+
+A saved SQL insight runs its own query text. A dashboard or insight date range reaches that
+text only through a ``{filters}`` placeholder, so a query with a hard-coded window ignores the
+date picker. A SQL chart also draws its rows in query order, so a time series needs an ascending
+``ORDER BY`` on its time column, or the newest bucket can show on the left.
+
+Each scorer grades the most recent successful ``insight-create`` or ``insight-update`` call that
+holds a ``HogQLQuery``.
+"""
+
+from __future__ import annotations
+
+import re
+import json
+from typing import Any
+
+from posthog.hogql import ast
+from posthog.hogql.errors import BaseHogQLError
+from posthog.hogql.functions.mapping import find_hogql_aggregation
+from posthog.hogql.parser import parse_select
+from posthog.hogql.visitor import TraversingVisitor
+
+from products.posthog_ai.eval_harness.log_parser import ToolCall
+from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer
+from products.posthog_ai.evals.cli_mcp.scorers import tool_output_text
+from products.posthog_ai.evals.product_analytics.scorers import _saved_query, parser_for
+
+_SAVE_TOOLS = ("insight-create", "insight-update")
+_CLOCK_FUNCTIONS = frozenset({"now", "now64", "today", "yesterday"})
+_TIME_BUCKET_FUNCTIONS = frozenset({"todate", "todatetime", "tomonday", "datetrunc", "date_trunc"})
+_RANGE_OPS = frozenset(
+ {
+ ast.CompareOperationOp.Gt,
+ ast.CompareOperationOp.GtEq,
+ ast.CompareOperationOp.Lt,
+ ast.CompareOperationOp.LtEq,
+ }
+)
+_OUTPUT_ID_RE = re.compile(r"^(?:id|short_id):\s*(\S+)\s*$", re.MULTILINE)
+
+
+def _saved_sql_calls(calls: list[ToolCall]) -> list[tuple[ToolCall, dict[str, Any]]]:
+ saved: list[tuple[ToolCall, dict[str, Any]]] = []
+ for call in calls:
+ answer = _saved_query(call)
+ if answer is not None and answer[0] == "execute-sql":
+ saved.append((call, answer[1]))
+ return saved
+
+
+def _last_saved_sql(output: dict | None) -> dict[str, Any] | None:
+ parser = parser_for(output)
+ if parser is None:
+ return None
+ saved = _saved_sql_calls(parser.get_tool_calls())
+ return saved[-1][1] if saved else None
+
+
+def _insight_ids(call: ToolCall) -> set[str]:
+ """The numeric id and short id of the insight that a save call wrote."""
+ if call.name == "insight-update":
+ return {str(call.input["id"])} if call.input.get("id") is not None else set()
+ text = tool_output_text(call.output)
+ try:
+ payload = json.loads(text)
+ except (TypeError, ValueError):
+ return set(_OUTPUT_ID_RE.findall(text))
+ if not isinstance(payload, dict):
+ return set()
+ return {str(payload[key]) for key in ("id", "short_id") if payload.get(key) is not None}
+
+
+def _parse(source: dict[str, Any]) -> ast.SelectQuery | ast.SelectSetQuery | None:
+ text = source.get("query")
+ if not isinstance(text, str) or not text.strip():
+ return None
+ try:
+ return parse_select(text)
+ except BaseHogQLError:
+ return None
+
+
+def _hogql_text(node: ast.Expr) -> str:
+ """The expression as HogQL for score metadata.
+
+ The printer cannot print a placeholder such as ``{filters.dateRange.from}``, and a failed
+ print must not turn a graded case into a scorer error.
+ """
+ try:
+ return node.to_hogql()
+ except BaseHogQLError:
+ return f"<{type(node).__name__} with a placeholder>"
+
+
+def _outer_select(node: ast.SelectQuery | ast.SelectSetQuery) -> ast.SelectQuery:
+ while isinstance(node, ast.SelectSetQuery):
+ node = node.initial_select_query
+ return node
+
+
+class _Calls(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.names: set[str] = set()
+ self.raw_names: set[str] = set()
+ self.fields: set[str] = set()
+
+ def visit_call(self, node: ast.Call) -> None:
+ self.names.add(node.name.lower())
+ self.raw_names.add(node.name)
+ super().visit_call(node)
+
+ def visit_field(self, node: ast.Field) -> None:
+ if node.chain:
+ self.fields.add(str(node.chain[-1]).lower())
+ super().visit_field(node)
+
+
+def _calls_in(node: ast.Expr) -> _Calls:
+ found = _Calls()
+ found.visit(node)
+ return found
+
+
+class _DateLiterals(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.found = False
+
+ def visit_constant(self, node: ast.Constant) -> None:
+ if isinstance(node.value, str) and node.value[:4].isdigit():
+ self.found = True
+
+
+def _has_date_literal(node: ast.Expr) -> bool:
+ literals = _DateLiterals()
+ literals.visit(node)
+ return literals.found
+
+
+class _HardCodedWindows(TraversingVisitor):
+ """Range comparisons against the clock or a date literal, such as ``timestamp >= now() - INTERVAL 7 DAY``.
+
+ A literal counts inside a conversion too, such as ``timestamp >= toDateTime('2026-01-01')``.
+ """
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.found: list[str] = []
+
+ def _check(self, node: ast.Expr, bounds: list[ast.Expr]) -> None:
+ if any(_calls_in(bound).names & _CLOCK_FUNCTIONS or _has_date_literal(bound) for bound in bounds):
+ self.found.append(_hogql_text(node))
+
+ def visit_compare_operation(self, node: ast.CompareOperation) -> None:
+ if node.op in _RANGE_OPS:
+ self._check(node, [node.left, node.right])
+ super().visit_compare_operation(node)
+
+ def visit_between_expr(self, node: ast.BetweenExpr) -> None:
+ self._check(node, [node.low, node.high])
+ super().visit_between_expr(node)
+
+
+class _DateBoundingPlaceholders(TraversingVisitor):
+ """Finds a ``filters`` placeholder that adds a date predicate.
+
+ ``{filters}``, ``{filters.dateRange.from}``, ``{filters.dateRange.to}``, and
+ ``{filters(<column> AS timestamp)}`` bound the rows. Other forms, such as ``{filters.interval}``,
+ only substitute a value, so a query that uses only them still reads all time.
+ """
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.found = False
+
+ def visit_placeholder(self, node: ast.Placeholder) -> None:
+ chain = node.chain
+ if chain == ["filters"] or (chain is not None and chain[:2] == ["filters", "dateRange"]):
+ self.found = True
+ elif isinstance(node.expr, ast.Call) and node.expr.name == "filters":
+ self.found = self.found or any(
+ isinstance(arg, ast.Alias)
+ and arg.alias == "timestamp"
+ and not (isinstance(arg.expr, ast.Constant) and arg.expr.value is None)
+ for arg in node.expr.args
+ )
+
+
+def _has_date_bounding_placeholder(node: ast.Expr) -> bool:
+ placeholders = _DateBoundingPlaceholders()
+ placeholders.visit(node)
+ return placeholders.found
+
+
+def _graded_sql(
+ name: str, output: dict | None, expected: dict | None
+) -> tuple[dict[str, Any], ast.SelectQuery | ast.SelectSetQuery] | Score:
+ """The saved SQL source and its parsed query, or the score to return when there is nothing to grade."""
+ if not isinstance(expected, dict) or not isinstance(expected.get(name), dict):
+ return Score(name=name, score=None, metadata={"reason": f"No {name} key on case"})
+ source = _last_saved_sql(output)
+ if source is None:
+ return Score(name=name, score=None, metadata={"reason": "No SQL insight was saved"})
+ node = _parse(source)
+ if node is None:
+ return Score(name=name, score=0.0, metadata={"reason": "The saved SQL does not parse"})
+ return source, node
+
+
+class SavedSqlInsight(Scorer):
+ """Binary: did the agent save a SQL insight, on a dashboard when the case asks for one?
+
+ ``expected = {"saved_sql_insight": {}}`` or ``{"saved_sql_insight": {"on_dashboard": True}}``
+
+ This is the manipulation check for the suite: the other scorers grade the saved SQL, so a
+ case without one tells nothing about date filters or ordering. ``on_dashboard`` passes when
+ the call that saved the graded SQL insight sets a non-empty ``dashboards`` list, or a later
+ ``insight-update`` of that same insight does. ``None`` if the spec is absent.
+ """
+
+ def _name(self) -> str:
+ return "saved_sql_insight"
+
+ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:
+ spec = expected.get(self._name()) if isinstance(expected, dict) else None
+ if not isinstance(spec, dict):
+ return Score(name=self._name(), score=None, metadata={"reason": f"No {self._name()} key on case"})
+ parser = parser_for(output)
+ if parser is None:
+ return Score(name=self._name(), score=0.0, metadata={"reason": "No raw log"})
+ calls = [call for call in parser.get_tool_calls() if not call.is_error and call.name in _SAVE_TOOLS]
+ saved = _saved_sql_calls(calls)
+ if not saved:
+ return Score(name=self._name(), score=0.0, metadata={"reason": "No SQL insight was saved"})
+ if spec.get("on_dashboard"):
+ save_call = saved[-1][0]
+ ids = _insight_ids(save_call)
+ on_dashboard = bool(save_call.input.get("dashboards")) or any(
+ call.name == "insight-update"
+ and call.position > save_call.position
+ and call.input.get("dashboards")
+ and _insight_ids(call) & ids
+ for call in calls
+ )
+ if not on_dashboard:
+ return Score(
+ name=self._name(),
+ score=0.0,
+ metadata={"reason": "The SQL insight was not added to a dashboard", "insight_ids": sorted(ids)},
+ )
+ return Score(name=self._name(), score=1.0)
+
+
+class SqlFollowsDateFilter(Scorer):
+ """Binary: does the saved SQL insight follow the dashboard and insight date range?
+
+ ``expected = {"sql_follows_date_filter": {}}``
+
+ Passes when all of these are true:
+
+ - The query has a date-bounding ``{filters}`` placeholder: ``{filters}``,
+ ``{filters(<column> AS timestamp)}``, or ``{filters.dateRange.from}``.
+ - The query has no hard-coded window, such as ``timestamp >= now() - INTERVAL 14 DAY``.
+ A hard-coded window still clips the rows when a dashboard sets a wider range.
+ - ``source.filters.dateRange.date_from`` is set. Without it, ``{filters}`` is ``TRUE`` and
+ the insight shows all time instead of the window the user asked for.
+
+ ``None`` when no SQL insight was saved (graded by ``saved_sql_insight``) or the spec is absent.
+ """
+
+ def _name(self) -> str:
+ return "sql_follows_date_filter"
+
+ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:
+ graded = _graded_sql(self._name(), output, expected)
+ if isinstance(graded, Score):
+ return graded
+ source, node = graded
+
+ windows = _HardCodedWindows()
+ windows.visit(node)
+ filters = source.get("filters")
+ date_range = filters.get("dateRange") if isinstance(filters, dict) else None
+ failures = []
+ if not _has_date_bounding_placeholder(node):
+ failures.append("no {filters} placeholder")
+ if windows.found:
+ failures.append("hard-coded date window")
+ if not (isinstance(date_range, dict) and date_range.get("date_from")):
+ failures.append("no default filters.dateRange.date_from")
+ return Score(
+ name=self._name(),
+ score=0.0 if failures else 1.0,
+ metadata={"failures": failures, "hard_coded_windows": windows.found, "date_range": date_range},
+ )
+
+
+def _is_time_bucket(node: ast.Expr) -> bool:
+ found = _calls_in(node)
+ return (
+ any(
+ name.startswith("tostartof") or name in _TIME_BUCKET_FUNCTIONS or name in _CLOCK_FUNCTIONS
+ for name in found.names
+ )
+ or "timestamp" in found.fields
+ )
+
+
+def _field_name(node: ast.Expr) -> str | None:
+ return str(node.chain[-1]).lower() if isinstance(node, ast.Field) and node.chain else None
+
+
+def _column_name(column: ast.Expr) -> str | None:
+ return column.alias.lower() if isinstance(column, ast.Alias) else _field_name(column)
+
+
+def _source_queries(
+ select: ast.SelectQuery, ctes: dict[str, ast.Expr]
+) -> list[tuple[ast.SelectQuery | ast.SelectSetQuery, dict[str, ast.Expr]]]:
+ """The subqueries and CTE bodies that ``select`` reads from, each with the CTE scope to resolve it in.
+
+ A CTE body is resolved without its own name, so ``WITH events AS (SELECT ... FROM events)``
+ reads the physical ``events`` table and does not refer back to itself.
+ """
+ sources: list[tuple[ast.SelectQuery | ast.SelectSetQuery, dict[str, ast.Expr]]] = []
+ join = select.select_from
+ while join is not None:
+ table: ast.Expr | None = join.table
+ scope = ctes
+ if isinstance(table, ast.Field) and len(table.chain) == 1:
+ name = str(table.chain[0]).lower()
+ table = ctes.get(name)
+ scope = {key: value for key, value in ctes.items() if key != name}
+ if isinstance(table, ast.SelectQuery | ast.SelectSetQuery):
+ sources.append((table, scope))
+ join = join.next_join
+ return sources
+
+
+def _time_columns(node: ast.SelectQuery | ast.SelectSetQuery, ctes: dict[str, ast.Expr]) -> set[str]:
+ """Output columns that hold a time bucket, followed through subqueries, joins, and CTEs.
+
+ Agents often fill empty buckets with a generated date series, such as
+ ``SELECT today() - number AS day FROM numbers(14)``, joined to the counts, and select the
+ bucket in the outer query only as ``d.day``.
+ """
+ select = _outer_select(node)
+ ctes = {**ctes, **{name.lower(): cte.expr for name, cte in (select.ctes or {}).items()}}
+ inner = set().union(*(_time_columns(source, scope) for source, scope in _source_queries(select, ctes)))
+ names = set()
+ for column in select.select or []:
+ expr = column.expr if isinstance(column, ast.Alias) else column
+ name = _column_name(column)
+ if name and (_is_time_bucket(expr) or _field_name(expr) in inner):
+ names.add(name)
+ return names
+
+
+def _is_aggregate(node: ast.Expr) -> bool:
+ return any(find_hogql_aggregation(name) for name in _calls_in(node).raw_names)
+
+
+def _order_key_kinds(node: ast.SelectQuery | ast.SelectSetQuery) -> list[tuple[ast.OrderExpr, str]]:
+ """Each outer ``ORDER BY`` key, classed as ``time``, ``value`` (an aggregate), or ``group``."""
+ select = _outer_select(node)
+ columns = list(select.select or [])
+ by_name = {name: column for column in columns if (name := _column_name(column))}
+ time_columns = _time_columns(node, {})
+ kinds = []
+ for order in select.order_by or []:
+ expr = order.expr
+ if isinstance(expr, ast.Constant) and isinstance(expr.value, int) and 0 < expr.value <= len(columns):
+ column: ast.Expr = columns[expr.value - 1]
+ else:
+ column = by_name.get(_field_name(expr) or "", expr)
+ if _column_name(column) in time_columns or _field_name(expr) in time_columns or _is_time_bucket(column):
+ kinds.append((order, "time"))
+ elif _is_aggregate(column):
+ kinds.append((order, "value"))
+ else:
+ kinds.append((order, "group"))
+ return kinds
+
+
+class SqlTimeAxisAscending(Scorer):
+ """Binary: does the saved SQL time series sort its time column from oldest to newest?
+
+ ``expected = {"sql_time_axis_ascending": {}}``
+
+ A SQL chart takes its X axis from the row order. The outer query must order by its time
+ bucket (by alias, position, or expression), and every time key must be ascending. A time
+ bucket selected from a subquery, a join, or a CTE counts as the time bucket. A breakdown
+ column can come before the time key, but an aggregate cannot, because ``ORDER BY uploads DESC, day``
+ sorts the rows by value first. A missing ``ORDER BY`` fails, because ClickHouse returns
+ grouped rows in no fixed order. ``None`` when no SQL insight was saved or the spec is absent.
+ """
+
+ def _name(self) -> str:
+ return "sql_time_axis_ascending"
+
+ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:
+ graded = _graded_sql(self._name(), output, expected)
+ if isinstance(graded, Score):
+ return graded
+ _, node = graded
+
+ kinds = _order_key_kinds(node)
+ time_positions = [index for index, (_, kind) in enumerate(kinds) if kind == "time"]
+ if not time_positions:
+ return Score(name=self._name(), score=0.0, metadata={"reason": "No ORDER BY on the time column"})
+ descending = [_hogql_text(order.expr) for order, kind in kinds if kind == "time" and order.order == "DESC"]
+ values_first = [_hogql_text(order.expr) for order, kind in kinds[: time_positions[0]] if kind == "value"]
+ return Score(
+ name=self._name(),
+ score=0.0 if descending or values_first else 1.0,
+ metadata={
+ "order_keys": [f"{_hogql_text(order)} ({kind})" for order, kind in kinds],
+ "descending": descending,
+ "values_before_time": values_first,
+ },
+ )
diff --git a/products/product_analytics/mcp/tools.yaml b/products/product_analytics/mcp/tools.yaml
index ce4e69c4a0ef..a0ecc253f7ea 100644
--- a/products/product_analytics/mcp/tools.yaml
+++ b/products/product_analytics/mcp/tools.yaml
@@ -78,6 +78,15 @@ tools:
so a query you just ran through one of the `query-*` tools can be passed straight through.
+ A saved SQL insight must follow the dashboard and insight date range. Before you save a `HogQLQuery`,
+ replace its fixed date window, such as `timestamp >= now() - INTERVAL 30 DAY`, with the `{filters}`
+ placeholder: `WHERE event = 'signed_up' AND {filters}`. Then put the requested window in
+ `query.source.filters.dateRange`, for example `{"date_from": "-30d"}`. A fixed window ignores the date
+ picker, and `{filters}` without a `dateRange` shows all time. `{filters}` also applies the property and
+ test-account filters. For a table without a `timestamp` column, bind its date column with
+ `{filters(created_at AS timestamp)}`.
+
+
Save a `MetricsQuery` or `MetricsHistogramQuery` bare, as the top-level `query`. Do not put it in an
`InsightVizNode`.
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index d506e261b7be..94d0efe8e4cf 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -8178,7 +8178,7 @@
}
},
"insight-create": {
- "description": "Create a new saved insight from a name and query definition. Test queries with query-trends / query-funnel /\nquery-retention / query-paths / query-stickiness / query-lifecycle first to confirm the shape, then save.\nReturns insight metadata only — after creating, call the insight-query tool with the returned `short_id` if\nyou want to see the computed results.\n\nPut the insight definition in `query.source`, under the wrapper node that matches it. Use\n`InsightVizNode` for a product analytics chart, `DataVisualizationNode` for a SQL visualization, and `DataTableNode` with an `EventsQuery` source for a table of individual events:\n\n```json\n{ \"name\": \"Example\", \"query\": { \"kind\": \"DataVisualizationNode\", \"source\": { \"kind\": \"HogQLQuery\", \"query\": \"SELECT 1\" } } }\n```\nBare `TrendsQuery`, `HogQLQuery`, and `EventsQuery` definitions are accepted too. They are wrapped for you, so a query you just ran through one of the `query-*` tools can be passed straight through.\n\nSave a `MetricsQuery` or `MetricsHistogramQuery` bare, as the top-level `query`. Do not put it in an `InsightVizNode`.\n\nChoose the TrendsQuery `trendsFilter.display` from the question the insight answers:\n- Change over time: `ActionsLineGraph` (default), `ActionsAreaGraph`, or `ActionsBar`.\n- Compare breakdown values or series as totals (\"top pages\", \"failures by reason\"): `ActionsBarValue`, or `ActionsPie` for the share of a whole with a few values.\n- One number for the period, or this period against the previous one: `Metric` with `compareFilter.compare: true` for a comparison, or `BoldNumber` when a trend has no meaning.\n- Many breakdown rows that a person reads rather than compares: `ActionsTable`.\n- Totals per country from a `$geoip_country_code` breakdown: `WorldMap`.\n\nIn a TrendsQuery, `ActionsBar` is a time series with one bar per interval and the breakdown values stacked in each bar. It is not one bar per category. In a SQL `DataVisualizationNode`, `ActionsBar` is one bar per X-axis value, and `ActionsBarValue` draws the same bars horizontally.",
+ "description": "Create a new saved insight from a name and query definition. Test queries with query-trends / query-funnel /\nquery-retention / query-paths / query-stickiness / query-lifecycle first to confirm the shape, then save.\nReturns insight metadata only — after creating, call the insight-query tool with the returned `short_id` if\nyou want to see the computed results.\n\nPut the insight definition in `query.source`, under the wrapper node that matches it. Use\n`InsightVizNode` for a product analytics chart, `DataVisualizationNode` for a SQL visualization, and `DataTableNode` with an `EventsQuery` source for a table of individual events:\n\n```json\n{ \"name\": \"Example\", \"query\": { \"kind\": \"DataVisualizationNode\", \"source\": { \"kind\": \"HogQLQuery\", \"query\": \"SELECT 1\" } } }\n```\nBare `TrendsQuery`, `HogQLQuery`, and `EventsQuery` definitions are accepted too. They are wrapped for you, so a query you just ran through one of the `query-*` tools can be passed straight through.\n\nA saved SQL insight must follow the dashboard and insight date range. Before you save a `HogQLQuery`, replace its fixed date window, such as `timestamp >= now() - INTERVAL 30 DAY`, with the `{filters}` placeholder: `WHERE event = 'signed_up' AND {filters}`. Then put the requested window in `query.source.filters.dateRange`, for example `{\"date_from\": \"-30d\"}`. A fixed window ignores the date picker, and `{filters}` without a `dateRange` shows all time. `{filters}` also applies the property and test-account filters. For a table without a `timestamp` column, bind its date column with `{filters(created_at AS timestamp)}`.\n\nSave a `MetricsQuery` or `MetricsHistogramQuery` bare, as the top-level `query`. Do not put it in an `InsightVizNode`.\n\nChoose the TrendsQuery `trendsFilter.display` from the question the insight answers:\n- Change over time: `ActionsLineGraph` (default), `ActionsAreaGraph`, or `ActionsBar`.\n- Compare breakdown values or series as totals (\"top pages\", \"failures by reason\"): `ActionsBarValue`, or `ActionsPie` for the share of a whole with a few values.\n- One number for the period, or this period against the previous one: `Metric` with `compareFilter.compare: true` for a comparison, or `BoldNumber` when a trend has no meaning.\n- Many breakdown rows that a person reads rather than compares: `ActionsTable`.\n- Totals per country from a `$geoip_country_code` breakdown: `WorldMap`.\n\nIn a TrendsQuery, `ActionsBar` is a time series with one bar per interval and the breakdown values stacked in each bar. It is not one bar per category. In a SQL `DataVisualizationNode`, `ActionsBar` is one bar per X-axis value, and `ActionsBarValue` draws the same bars horizontally.",
"category": "Product analytics",
"feature": "product_analytics",
"summary": "Create insight",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index 55ad06334846..0abd96fab2a8 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -8396,7 +8396,7 @@
}
},
"insight-create": {
- "description": "Create a new saved insight from a name and query definition. Test queries with query-trends / query-funnel /\nquery-retention / query-paths / query-stickiness / query-lifecycle first to confirm the shape, then save.\nReturns insight metadata only — after creating, call the insight-query tool with the returned `short_id` if\nyou want to see the computed results.\n\nPut the insight definition in `query.source`, under the wrapper node that matches it. Use\n`InsightVizNode` for a product analytics chart, `DataVisualizationNode` for a SQL visualization, and `DataTableNode` with an `EventsQuery` source for a table of individual events:\n\n```json\n{ \"name\": \"Example\", \"query\": { \"kind\": \"DataVisualizationNode\", \"source\": { \"kind\": \"HogQLQuery\", \"query\": \"SELECT 1\" } } }\n```\nBare `TrendsQuery`, `HogQLQuery`, and `EventsQuery` definitions are accepted too. They are wrapped for you, so a query you just ran through one of the `query-*` tools can be passed straight through.\n\nSave a `MetricsQuery` or `MetricsHistogramQuery` bare, as the top-level `query`. Do not put it in an `InsightVizNode`.\n\nChoose the TrendsQuery `trendsFilter.display` from the question the insight answers:\n- Change over time: `ActionsLineGraph` (default), `ActionsAreaGraph`, or `ActionsBar`.\n- Compare breakdown values or series as totals (\"top pages\", \"failures by reason\"): `ActionsBarValue`, or `ActionsPie` for the share of a whole with a few values.\n- One number for the period, or this period against the previous one: `Metric` with `compareFilter.compare: true` for a comparison, or `BoldNumber` when a trend has no meaning.\n- Many breakdown rows that a person reads rather than compares: `ActionsTable`.\n- Totals per country from a `$geoip_country_code` breakdown: `WorldMap`.\n\nIn a TrendsQuery, `ActionsBar` is a time series with one bar per interval and the breakdown values stacked in each bar. It is not one bar per category. In a SQL `DataVisualizationNode`, `ActionsBar` is one bar per X-axis value, and `ActionsBarValue` draws the same bars horizontally.",
+ "description": "Create a new saved insight from a name and query definition. Test queries with query-trends / query-funnel /\nquery-retention / query-paths / query-stickiness / query-lifecycle first to confirm the shape, then save.\nReturns insight metadata only — after creating, call the insight-query tool with the returned `short_id` if\nyou want to see the computed results.\n\nPut the insight definition in `query.source`, under the wrapper node that matches it. Use\n`InsightVizNode` for a product analytics chart, `DataVisualizationNode` for a SQL visualization, and `DataTableNode` with an `EventsQuery` source for a table of individual events:\n\n```json\n{ \"name\": \"Example\", \"query\": { \"kind\": \"DataVisualizationNode\", \"source\": { \"kind\": \"HogQLQuery\", \"query\": \"SELECT 1\" } } }\n```\nBare `TrendsQuery`, `HogQLQuery`, and `EventsQuery` definitions are accepted too. They are wrapped for you, so a query you just ran through one of the `query-*` tools can be passed straight through.\n\nA saved SQL insight must follow the dashboard and insight date range. Before you save a `HogQLQuery`, replace its fixed date window, such as `timestamp >= now() - INTERVAL 30 DAY`, with the `{filters}` placeholder: `WHERE event = 'signed_up' AND {filters}`. Then put the requested window in `query.source.filters.dateRange`, for example `{\"date_from\": \"-30d\"}`. A fixed window ignores the date picker, and `{filters}` without a `dateRange` shows all time. `{filters}` also applies the property and test-account filters. For a table without a `timestamp` column, bind its date column with `{filters(created_at AS timestamp)}`.\n\nSave a `MetricsQuery` or `MetricsHistogramQuery` bare, as the top-level `query`. Do not put it in an `InsightVizNode`.\n\nChoose the TrendsQuery `trendsFilter.display` from the question the insight answers:\n- Change over time: `ActionsLineGraph` (default), `ActionsAreaGraph`, or `ActionsBar`.\n- Compare breakdown values or series as totals (\"top pages\", \"failures by reason\"): `ActionsBarValue`, or `ActionsPie` for the share of a whole with a few values.\n- One number for the period, or this period against the previous one: `Metric` with `compareFilter.compare: true` for a comparison, or `BoldNumber` when a trend has no meaning.\n- Many breakdown rows that a person reads rather than compares: `ActionsTable`.\n- Totals per country from a `$geoip_country_code` breakdown: `WorldMap`.\n\nIn a TrendsQuery, `ActionsBar` is a time series with one bar per interval and the breakdown values stacked in each bar. It is not one bar per category. In a SQL `DataVisualizationNode`, `ActionsBar` is one bar per X-axis value, and `ActionsBarValue` draws the same bars horizontally.",
"category": "Product analytics",
"feature": "product_analytics",
"summary": "Create insight",