Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(posthog-ai): add evals for sql routing misses and chart answers (#119445)

Daniel Parkerf8c9518eb20e
diff --git a/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py b/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py
index d32f20aaafcd..de14426be006 100644
--- a/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py
+++ b/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py
@@ -1,10 +1,19 @@
from __future__ import annotations
import json
+from urllib.parse import quote
from parameterized import parameterized
-from products.posthog_ai.evals.cli_mcp.scorers import DidNotCiteRawSqlAfterTypedQuery, FirstRelevantTool
+from products.posthog_ai.eval_harness.test.test_eval_scorers import _raw_tool_log
+from products.posthog_ai.evals.cli_mcp.scorers import (
+ INLINE_VIEW_NOTE,
+ ChartFromTypedQuery,
+ ChartShown,
+ DidNotCiteRawSqlAfterTypedQuery,
+ FirstRelevantTool,
+ NoMarkdownDataTable,
+)
def _tool_call_start(call_id: str, command: str) -> str:
@@ -241,3 +250,153 @@ def test_first_relevant_tool_turn_skips_a_missing_slice() -> None:
assert result.score is None
assert "No turn 9 log slice" in result.metadata["reason"]
+
+
+_TRENDS_RESULT: tuple[str, dict[str, list[str]], str] = ("mcp__posthog__query-trends", {"series": []}, "rows")
+_TRENDS_INLINE_RESULT: tuple[str, dict[str, list[str]], str] = (
+ "mcp__posthog__query-trends",
+ {"series": []},
+ f"rows\n\nThe user {INLINE_VIEW_NOTE}.",
+)
+_BASE = "https://us.example.com/project/1"
+_SQL_CHART = f"[Weekly signups]({_BASE}/sql?open_query=SELECT%20toStartOfWeek(timestamp)%2C%20count()%20FROM%20events)"
+_INSIGHT_CHART = f"[Weekly signups]({_BASE}/insights/9pQx3)"
+_HOGQL_NODE = '{"kind": "DataTableNode", "source": {"kind": "HogQLQuery", "query": "SELECT 1"}}'
+_TRENDS_NODE = '{"kind": "InsightVizNode", "source": {"kind": "TrendsQuery", "series": []}}'
+_PROJECT_RESULT: tuple[str, dict[str, list[str]], str] = (
+ "mcp__posthog__query-trends",
+ {"series": []},
+ f"rows\n{_BASE}/insights/new",
+)
+_UNSAVED_INSIGHT_LINK = f"[Weekly signups]({_BASE}/insights/new#q=%7B%7D)"
+_SAVED_TRENDS: tuple[str, dict[str, str], str] = (
+ "mcp__posthog__insight-get",
+ {"id": "9pQx3"},
+ '{"short_id": "9pQx3", "query": {"kind": "InsightVizNode", "source": {"kind": "TrendsQuery"}}}',
+)
+_SAVED_HOGQL: tuple[str, dict[str, str], str] = (
+ "mcp__posthog__insight-get",
+ {"id": "9pQx3"},
+ "short_id: 9pQx3\nquery:\n kind: DataVisualizationNode\n source:\n kind: HogQLQuery",
+)
+
+
+def _render(tool_name: str, status: str = "completed", tool_input: dict | None = None) -> tuple:
+ rendered = {"series": []} if tool_input is None else tool_input
+ return ("mcp__posthog__render-ui", {"tool_name": tool_name, "tool_input": rendered}, "Rendering", status)
+
+
+def _chart_output(calls: list[tuple], answer: str = "Signups rose.") -> dict:
+ return {
+ "raw_log": _raw_tool_log(calls),
+ "prompt": "Weekly signups",
+ "messages": [{"role": "assistant", "content": [{"type": "text", "text": answer}]}],
+ }
+
+
+ [
+ ("rendered_query_tool", [_TRENDS_RESULT, _render("query-trends")], "Signups rose.", 1.0, 1.0),
+ ("inline_view_note", [_TRENDS_INLINE_RESULT], "Signups rose.", 1.0, 1.0),
+ (
+ "saved_insight_link",
+ [_TRENDS_RESULT, _SAVED_TRENDS],
+ f"Signups rose.\n\n{_INSIGHT_CHART}\n\nMore text.",
+ 1.0,
+ 1.0,
+ ),
+ ("saved_hogql_insight_link", [_TRENDS_RESULT, _SAVED_HOGQL], f"Signups rose.\n\n{_INSIGHT_CHART}", 1.0, 0.0),
+ ("saved_insight_of_unknown_kind", [_TRENDS_RESULT], f"Signups rose.\n\n{_INSIGHT_CHART}", 1.0, None),
+ ("sql_link_after_typed_query", [_TRENDS_RESULT], f"Signups rose.\n\n{_SQL_CHART}", 1.0, 0.0),
+ ("sql_link_inside_sentence", [_TRENDS_RESULT], f"See {_SQL_CHART} for details.", 0.0, None),
+ ("unsaved_insight_link", [_TRENDS_RESULT], f"Signups rose.\n\n{_UNSAVED_INSIGHT_LINK}", 0.0, None),
+ (
+ "unsaved_trends_link",
+ [_TRENDS_RESULT],
+ f"Up.\n\n[Signups]({_BASE}/insights/new#q={quote(_TRENDS_NODE)})",
+ 0.0,
+ None,
+ ),
+ (
+ "unsaved_hogql_link",
+ [_TRENDS_RESULT],
+ f"Up.\n\n[Signups]({_BASE}/insights/new#q={quote(_HOGQL_NODE)})",
+ 1.0,
+ 0.0,
+ ),
+ ("short_insight_link", [_TRENDS_RESULT, _SAVED_TRENDS], f"Up.\n\n[Signups]({_BASE}/i/9pQx3)", 1.0, 1.0),
+ ("same_project_link", [_PROJECT_RESULT, _SAVED_TRENDS], f"Up.\n\n{_INSIGHT_CHART}", 1.0, 1.0),
+ (
+ "other_project_link",
+ [_PROJECT_RESULT],
+ "Up.\n\n[Signups](https://us.example.com/project/2/insights/9pQx3)",
+ 0.0,
+ None,
+ ),
+ ("failed_render", [_TRENDS_RESULT, _render("query-trends", status="failed")], "Signups rose.", 0.0, None),
+ ("rendered_other_tool", [_TRENDS_RESULT, _render("experiment-list")], "Signups rose.", 0.0, None),
+ (
+ "render_with_empty_input",
+ [_TRENDS_RESULT, _render("query-trends", tool_input={})],
+ "Signups rose.",
+ 0.0,
+ None,
+ ),
+ (
+ "render_with_other_input",
+ [_TRENDS_RESULT, _render("query-trends", tool_input={"series": [{"event": "x"}]})],
+ "Signups rose.",
+ 0.0,
+ None,
+ ),
+ (
+ "block_tag_alone",
+ [_TRENDS_RESULT, _SAVED_TRENDS],
+ 'Signups rose.\n\n<insight id="9pQx3" display="block"/>',
+ 1.0,
+ 1.0,
+ ),
+ ("block_tag_in_sentence", [_TRENDS_RESULT], 'See <insight id="9pQx3" display="block"/> here.', 0.0, None),
+ ("block_tag_without_id", [_TRENDS_RESULT], 'Signups rose.\n\n<insight display="block"/>', 0.0, None),
+ (
+ "block_tag_in_code_fence",
+ [_TRENDS_RESULT],
+ 'Use this tag:\n\n```\n<insight id="9pQx3" display="block"/>\n```',
+ 0.0,
+ None,
+ ),
+ ]
+)
+def test_chart_scorers_grade_each_chart_path(
+ _name: str, calls: list[tuple], answer: str, shown: float, typed: float | None
+) -> None:
+ output = _chart_output(calls, answer)
+ spec = {"tool_name_any_of": ["query-trends"]}
+
+ assert ChartShown()._run_eval_sync(output, expected={"chart_shown": spec}).score == shown
+ assert ChartFromTypedQuery()._run_eval_sync(output, expected={"chart_from_typed_query": spec}).score == typed
+
+
+_LONG_TABLE = "| Week | Signups |\n|---|---|\n| W1 | 10 |\n| W2 | 12 |\n| W3 | 9 |\n| W4 | 14 |"
+_SUMMARY_TABLE = "| Period | Signups |\n| :--- | ---: |\n| Last 7 days | 45 |\n| Previous 7 days | 38 |"
+
+
+ [
+ ("long_table", [f"Signups rose.\n\n{_LONG_TABLE}\n\nThey peaked in W4."], 0.0),
+ ("single_column_table", ["Weekly signups:\n\n| Signups |\n| --- |\n| 10 |\n| 12 |\n| 9 |\n| 14 |"], 0.0),
+ ("dashes_under_piped_line", ["a | b\n---\nSignups rose."], 1.0),
+ ("long_table_short_delimiters", [f"Signups rose.\n\n{_LONG_TABLE.replace('|---|---|', '| :- | -: |')}"], 0.0),
+ ("long_table_before_closing_line", [f"Signups rose.\n\n{_LONG_TABLE}", "I added two buttons above."], 0.0),
+ ("summary_table", [f"Signups rose 18%.\n\n{_SUMMARY_TABLE}"], 1.0),
+ ("prose_with_pipes", ["Signups rose 18% | weekly peak in W4 | no drop."], 1.0),
+ ]
+)
+def test_no_markdown_data_table_flags_long_tables(_name: str, texts: list[str], score: float) -> None:
+ messages = [{"role": "assistant", "content": [{"type": "text", "text": text}]} for text in texts]
+
+ result = NoMarkdownDataTable()._run_eval_sync(
+ {"messages": messages, "last_message": texts[-1]}, expected={"no_markdown_data_table": {}}
+ )
+
+ assert result.score == score
diff --git a/products/posthog_ai/eval_harness/test/test_visualization_choice_scorers.py b/products/posthog_ai/eval_harness/test/test_visualization_choice_scorers.py
index 87a33af9225a..1f7bfa798508 100644
--- a/products/posthog_ai/eval_harness/test/test_visualization_choice_scorers.py
+++ b/products/posthog_ai/eval_harness/test/test_visualization_choice_scorers.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import pytest
from products.posthog_ai.eval_harness.test.test_eval_scorers import _raw_tool_log
-from products.posthog_ai.evals.product_analytics.scorers import InsightShape
+from products.posthog_ai.evals.product_analytics.scorers import AnswerFromTypedQuery, InsightShape
def _output(calls) -> dict:
@@ -54,6 +54,30 @@ UPLOADS_BY_FILE_TYPE = {
}
+def _scorecard_call(maths: list[str | None], formula: bool = False, test_accounts: bool | None = None) -> tuple:
+ series = [{"kind": "EventsNode", "event": "uploaded_file", **({"math": m} if m else {})} for m in maths]
+ query: dict = {"series": series}
+ if formula:
+ query["trendsFilter"] = {"formulaNodes": [{"formula": "A-B"}]}
+ if test_accounts is not None:
+ query["filterTestAccounts"] = test_accounts
+ return ("mcp__posthog__query-trends", query, "ok")
+
+
+_NODES = [{"kind": "EventsNode", "event": "uploaded_file"}, {"kind": "EventsNode", "event": "downloaded_file"}]
+_DAU_NODES = [{**node, "math": "dau"} for node in _NODES]
+
+
+def _filtered_call(series_key: str | None = None, query_key: str | None = None) -> tuple:
+ node: dict = {"kind": "EventsNode", "event": "uploaded_file"}
+ if series_key:
+ node["properties"] = [{"key": series_key, "operator": "is_set", "type": "event"}]
+ query: dict = {"series": [node]}
+ if query_key:
+ query["properties"] = {"type": "AND", "values": [{"key": query_key, "value": "/pricing/", "type": "event"}]}
+ return ("mcp__posthog__query-trends", query, "ok")
+
+
def _sql_call(status: str = "completed") -> tuple:
return ("mcp__posthog__execute-sql", {"query": "SELECT 1"}, "1", status)
@@ -209,6 +233,28 @@ PAGEVIEWS_BY_BROWSER = {
0.0,
None,
),
+ ([_scorecard_call(["dau", None])], {"math": ["total", "dau"]}, 1.0, []),
+ ([_scorecard_call([None, None])], {"math": ["total", "dau"]}, 0.0, ["math"]),
+ ([_scorecard_call(["median", "p95"])], {"math": ["p95"]}, 1.0, []),
+ ([_scorecard_call(["dau", None], formula=True)], {"formula": True}, 1.0, []),
+ ([_scorecard_call(["dau", None])], {"formula": True}, 0.0, ["formula"]),
+ ([_scorecard_call(["dau"], test_accounts=True)], {"filter_test_accounts": True}, 1.0, []),
+ ([_scorecard_call(["dau"])], {"filter_test_accounts": True}, 0.0, ["filter_test_accounts"]),
+ (
+ [("mcp__posthog__query-trends", {"series": [{"kind": "GroupNode", "math": "dau", "nodes": _NODES}]}, "ok")],
+ {"math": ["dau"]},
+ 1.0,
+ [],
+ ),
+ (
+ [("mcp__posthog__query-trends", {"series": [{"kind": "GroupNode", "nodes": _DAU_NODES}]}, "ok")],
+ {"math": ["dau"]},
+ 0.0,
+ ["math"],
+ ),
+ ([_filtered_call(series_key="file_name")], {"filtered_on": ["file_name"]}, 1.0, []),
+ ([_filtered_call(query_key="$pathname")], {"filtered_on": ["$pathname", "$current_url"]}, 1.0, []),
+ ([_filtered_call()], {"filtered_on": ["file_name"]}, 0.0, ["filtered_on"]),
],
)
def test_insight_shape_checks_only_the_expected_keys(calls, shape, score, mismatched) -> None:
@@ -220,3 +266,69 @@ def test_insight_shape_checks_only_the_expected_keys(calls, shape, score, mismat
def test_insight_shape_skips_without_expectation() -> None:
assert InsightShape()._run_eval_sync(_output([_trends_call()]), {}).score is None
+
+
+def _result(tool: str, text: str, query: dict | None = None) -> tuple:
+ return (f"mcp__posthog__{tool}", query or {}, {"content": [{"type": "text", "text": text}]})
+
+
+_TYPED = _result("query-trends", "Date range|Uploads|Unique uploaders\n2026-10-12 to 2026-10-19|21|917")
+_SQL = _result("execute-sql", "uploads|downloads\n20|846", {"query": "SELECT count() FROM events"})
+_CATALOG = _result("execute-sql", "id|name\n42|Weekly logins", {"query": "SELECT id, name FROM system.insights"})
+_SQL_LINK = "[Uploads](https://us.example.com/project/1/sql?open_query=SELECT%20count()%20FROM%20events)"
+
+
+ "calls,answer,score",
+ [
+ ([_TYPED], "Oct 12 to Oct 19: 21 uploads and 917 unique uploaders.", 1.0),
+ (
+ [_TYPED, _SQL],
+ f"There were 21 uploads and 917 downloads.\n\n{_SQL_LINK}\n\nA rolling window gives 20 and 846.",
+ 1.0,
+ ),
+ ([_TYPED, _SQL], "There were 20 uploads and 846 downloads.", 0.0),
+ ([_SQL], "There were 20 uploads and 846 downloads.", 0.0),
+ ([_CATALOG, _TYPED], "No saved insight exists. There were 21 uploads.", 1.0),
+ ([_CATALOG], "Insight 42 already tracks weekly logins.", None),
+ ([_result("execute-sql", "n\n846", {"query": "SELECT count() FROM system.insights, events"})], "846", 0.0),
+ (
+ [_result("execute-sql", "n\n846", {"query": "SELECT count() FROM `events` JOIN system.insights i ON 1"})],
+ "846",
+ 0.0,
+ ),
+ ([_TYPED, _SQL], "Uploads rose by a third.", None),
+ ([_result("query-trends", "Net|-42"), _result("execute-sql", "net\n42")], "Net upgrades: 42.", 0.0),
+ ([_result("query-trends", "Net|-42"), _result("execute-sql", "net\n42")], "Net upgrades: -42.", 1.0),
+ (
+ [
+ _result("query-funnel", "Step|Conversion\nUploaded file|45\n\nExample: 90% of users convert."),
+ _result("execute-sql", "rate\n90"),
+ ],
+ "Conversion is 90%.",
+ 0.0,
+ ),
+ ([_result("query-trends", "Signups|2026")], "There were 2,026 signups.", 1.0),
+ (
+ [
+ _result(
+ "query-trends",
+ '{"query": {"trendsFilter": {"smoothingIntervals": 30}}, "results": [{"label": "x", "count": 917}]}',
+ ),
+ _result("execute-sql", "users\n30"),
+ ],
+ "30 users uploaded a file.",
+ 0.0,
+ ),
+ ],
+)
+def test_answer_from_typed_query_traces_reported_values(calls, answer, score) -> None:
+ output = {
+ "raw_log": _raw_tool_log(calls),
+ "prompt": "How many uploads in the last 7 days?",
+ "messages": [{"role": "assistant", "content": [{"type": "text", "text": answer}]}],
+ }
+
+ result = AnswerFromTypedQuery()._run_eval_sync(output, {"answer_from_typed_query": {}})
+
+ assert result.score == score
diff --git a/products/posthog_ai/evals/cli_mcp/scorers.py b/products/posthog_ai/evals/cli_mcp/scorers.py
index 30b6e2876e65..6e57f9e10b3a 100644
--- a/products/posthog_ai/evals/cli_mcp/scorers.py
+++ b/products/posthog_ai/evals/cli_mcp/scorers.py
@@ -31,11 +31,24 @@ unrelated cases don't drag the rollup down.
before any successful ``query-*`` call?
* ``DidNotCiteRawSqlAfterTypedQuery`` — after a typed query tool succeeded,
did the final answer avoid hand-writing the equivalent raw SQL?
+* ``ChartShown`` — did the user see the query result as a chart, through
+ ``render-ui``, the client's inline view, or a chart link in the answer?
+* ``ChartFromTypedQuery`` — was that chart the typed query, not rewritten SQL?
+* ``NoMarkdownDataTable`` — did the agent avoid retyping the result as a long
+ Markdown table?
"""
from __future__ import annotations
import re
+import json
+from collections.abc import Mapping
+from urllib.parse import parse_qs, urlsplit
+
+from posthog.hogql import ast
+from posthog.hogql.errors import BaseHogQLError
+from posthog.hogql.parser import parse_select
+from posthog.hogql.visitor import TraversingVisitor
from products.posthog_ai.backend.exec_commands import INFO_SYNTHETIC_PREFIX
from products.posthog_ai.eval_harness.log_parser import EXEC_TOOL_NAME, LogParser, ToolCall, is_schema_discovery_call
@@ -43,12 +56,15 @@ from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer
__all__ = [
"CalledTargetTool",
+ "ChartFromTypedQuery",
+ "ChartShown",
"DidNotCiteRawSqlAfterTypedQuery",
"DidNotRenderUi",
"DrilledIntoSchema",
"ExecBeforeRender",
"InfoBeforeCall",
"FirstRelevantTool",
+ "NoMarkdownDataTable",
"PreferredSearchOverTools",
"RanPythonPostProcessing",
"RecoveredToCorrectTool",
@@ -1088,3 +1104,407 @@ class DidNotRenderUi(Scorer):
score=1.0,
metadata={"scope": "any render-ui" if target_set is None else sorted(target_set)},
)
+
+
+# Mirrors ``UI_APP_RENDER_NOTE`` in ``services/mcp/src/lib/build-tool-result.ts``: the MCP server appends it to a
+# query result when the client already displays that result's UI app inline.
+INLINE_VIEW_NOTE = "already sees this result as an interactive view"
+
+# A Markdown link alone in its paragraph, which PostHog Desktop draws as a full-size chart card.
+# The URL runs to the closing parenthesis because SQL in ``open_query`` keeps unencoded ``(`` and ``)``.
+_CHART_LINK_RE = re.compile(r'^\[[^\]]+\]\((https?://\S+?)(?:\s+"[^"]*")?\)$')
+# A block-display object tag alone in its paragraph, the other chart syntax in
+# ``products/desktop/packages/ui/src/utils/remarkObjectTags.ts``. A tag inside a sentence renders as a chip.
+_CHART_TAG_RE = re.compile(r"^<(insight|hogql)\b([^>]*?)/?>(.*?)(?:</\1>)?$", re.IGNORECASE | re.DOTALL)
+_TAG_ATTR_RE = re.compile(r'(\w+)="([^"]*)"')
+_CODE_FENCE_RE = re.compile(r"^(```|~~~).*?^\1[^\n]*$", re.MULTILINE | re.DOTALL)
+_PROJECT_URL_RE = re.compile(r"/project/(\d+)/")
+
+
+def assistant_text(output: dict | None) -> str:
+ """All text the agent wrote to the user, across every assistant message.
+
+ ``last_message`` is only the final text block, which is often a closing line written after
+ a follow-up tool call, so presentation checks read the whole conversation instead.
+ """
+ texts: list[str] = []
+ for message in (output or {}).get("messages") or []:
+ if not isinstance(message, dict) or message.get("role") != "assistant":
+ continue
+ content = message.get("content")
+ if isinstance(content, str):
+ texts.append(content)
+ elif isinstance(content, list):
+ texts.extend(
+ block["text"]
+ for block in content
+ if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str)
+ )
+ if not texts:
+ last_message = (output or {}).get("last_message")
+ if isinstance(last_message, str):
+ texts.append(last_message)
+ return "\n\n".join(text for text in texts if text)
+
+
+def tool_output_text(output: str) -> str:
+ """The text a tool returned, without the MCP content blocks that the log keeps around it."""
+ try:
+ payload = json.loads(output)
+ except (TypeError, ValueError):
+ return output
+ if isinstance(payload, dict):
+ payload = payload.get("content")
+ if not isinstance(payload, list):
+ return output
+ texts = [
+ block["text"]
+ for block in payload
+ if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str)
+ ]
+ return "\n".join(texts) if texts else output
+
+
+def _query_node_kind(node: object) -> str | None:
+ """``sql`` for a ``HogQLQuery`` at any wrapper depth, ``insight`` for any other query kind."""
+ for _ in range(3):
+ if not isinstance(node, dict):
+ return None
+ kind = node.get("kind")
+ if kind == "HogQLQuery":
+ return "sql"
+ if not isinstance(node.get("source"), dict):
+ return "insight" if isinstance(kind, str) and kind else None
+ node = node["source"]
+ return None
+
+
+def _collect_saved_insight_kinds(node: object, kinds: dict[str, str]) -> None:
+ if isinstance(node, dict):
+ short_id = node.get("short_id")
+ kind = _query_node_kind(node.get("query"))
+ if isinstance(short_id, str) and short_id and kind is not None:
+ kinds[short_id] = kind
+ for value in node.values():
+ _collect_saved_insight_kinds(value, kinds)
+ elif isinstance(node, list):
+ for value in node:
+ _collect_saved_insight_kinds(value, kinds)
+
+
+# Both match JSON and the TOON text that exec returns in its default output format.
+_SHORT_ID_RE = re.compile(r"""(?:\bshort_id["']?\s*:\s*["']?|/insights/)(\w+)""")
+_QUERY_KIND_RE = re.compile(r"""\bkind["']?\s*:\s*["']?(\w+)""")
+
+
+def saved_insight_kinds(calls: list[ToolCall]) -> dict[str, str]:
+ """The query kind (``insight`` or ``sql``) of each saved insight whose query a tool call showed.
+
+ A saved insight can hold a ``HogQLQuery``, so its link alone does not tell a typed chart from a
+ SQL chart. JSON output is read exactly. Text output is read only when it names one insight.
+ """
+ kinds: dict[str, str] = {}
+ for call in calls:
+ if call.is_error:
+ continue
+ text = tool_output_text(call.output)
+ try:
+ _collect_saved_insight_kinds(json.loads(text), kinds)
+ except (TypeError, ValueError):
+ pass
+ short_ids = set(_SHORT_ID_RE.findall(text)) - {"new"}
+ if len(short_ids) != 1 or next(iter(short_ids)) in kinds:
+ continue
+ query = call.input.get("query")
+ query_kinds = set(_QUERY_KIND_RE.findall(text + (json.dumps(query) if isinstance(query, dict) else "")))
+ if "HogQLQuery" in query_kinds:
+ kinds[next(iter(short_ids))] = "sql"
+ elif any(kind.endswith("Query") for kind in query_kinds):
+ kinds[next(iter(short_ids))] = "insight"
+ return kinds
+
+
+def _chart_tag_kind(paragraph: str, insight_kinds: Mapping[str, str]) -> str | None:
+ match = _CHART_TAG_RE.match(paragraph)
+ if match is None:
+ return None
+ tag, attrs, body = match.group(1).lower(), dict(_TAG_ATTR_RE.findall(match.group(2))), match.group(3).strip()
+ if attrs.get("display") != "block":
+ return None
+ if tag == "insight":
+ short_id = attrs.get("id")
+ return insight_kinds.get(short_id, "saved") if short_id else None
+ return "sql" if body else None
+
+
+def chart_references(
+ text: str, project_ids: frozenset[str] = frozenset(), insight_kinds: Mapping[str, str] | None = None
+) -> list[str]:
+ """Kinds of the full-size charts in ``text``, in order.
+
+ ``insight`` is a typed query, ``sql`` is a SQL query, and ``saved`` is a saved insight that is
+ not in ``insight_kinds``, so its query kind is unknown. Only a paragraph that holds nothing but a
+ chart link or a block tag counts, and code blocks are skipped, because Desktop draws nothing else
+ as a chart card. When ``project_ids`` is not empty, a link to any other project is skipped.
+ """
+ insight_kinds = insight_kinds or {}
+ kinds: list[str] = []
+ for paragraph in re.split(r"\n\s*\n", _CODE_FENCE_RE.sub("", text)):
+ paragraph = paragraph.strip()
+ match = _CHART_LINK_RE.match(paragraph)
+ if match is None:
+ tag_kind = _chart_tag_kind(paragraph, insight_kinds)
+ if tag_kind is not None:
+ kinds.append(tag_kind)
+ continue
+ link_kind = _chart_link_kind(match.group(1), project_ids, insight_kinds)
+ if link_kind is not None:
+ kinds.append(link_kind)
+ return kinds
+
+
+def _hogql_from_query_node(raw: str) -> str | None:
+ try:
+ node = json.loads(raw)
+ except ValueError:
+ return None
+ for _ in range(3):
+ if not isinstance(node, dict):
+ return None
+ if node.get("kind") == "HogQLQuery" and isinstance(node.get("query"), str):
+ return node["query"]
+ node = node.get("source")
+ return None
+
+
+def _chart_link_kind(url: str, project_ids: frozenset[str], insight_kinds: Mapping[str, str]) -> str | None:
+ """Mirror ``parsePostHogObjectUrl`` in ``products/desktop/packages/core/src/posthog-objects/objectUrls.ts``.
+
+ A saved insight (``/insights/<id>``, ``/i/<id>``) takes its kind from ``insight_kinds``. ``/sql?open_query=`` and an
+ ``/insights/new#q=`` link that holds a ``HogQLQuery`` are SQL charts; an unsaved typed query is no
+ chart. Desktop also requires the app host, which the eval cannot know because the agent's links
+ and the tool results use different hosts, so only the project is checked.
+ """
+ parts = urlsplit(url)
+ segments = [segment for segment in parts.path.split("/") if segment]
+ if segments[:1] == ["project"]:
+ if len(segments) < 2 or (project_ids and segments[1] not in project_ids):
+ return None
+ segments = segments[2:]
+ if segments == ["sql"]:
+ query = parse_qs(parts.query).get("open_query", [""])[0].strip()
+ if query.startswith("{"):
+ query = (_hogql_from_query_node(query) or "").strip()
+ return "sql" if query else None
+ if segments == ["insights", "new"]:
+ node = parse_qs(parts.fragment).get("q", [""])[0]
+ return "sql" if (_hogql_from_query_node(node) or "").strip() else None
+ if len(segments) >= 2 and segments[0] in ("insights", "i") and segments[1] != "new":
+ return insight_kinds.get(segments[1], "saved")
+ return None
+
+
+# Inputs the exec wrapper accepts that are not part of the query, so a render that leaves them out still matches.
+_NON_QUERY_INPUT_KEYS = frozenset({"output_format"})
+
+
+def _query_input(call: ToolCall) -> dict:
+ return {key: value for key, value in call.input.items() if key not in _NON_QUERY_INPUT_KEYS}
+
+
+def _render_matches_query(render: ToolCall, calls: list[ToolCall]) -> bool:
+ """True when a ``render-ui`` call passes the input of a successful call to the same query tool.
+
+ ``render-ui`` succeeds for any input because it only emits a render directive, and the chart then
+ fetches its own data. An empty or invented input therefore shows an error, not a chart.
+ """
+ tool_name = _rendered_tool_name(render)
+ tool_input = render.input.get("tool_input")
+ if not isinstance(tool_input, dict) or not tool_input:
+ return False
+ rendered = {key: value for key, value in tool_input.items() if key not in _NON_QUERY_INPUT_KEYS}
+ return any(
+ not call.is_error and call.name == tool_name and _query_input(call) == rendered
+ for call in calls
+ if call.position < render.position
+ )
+
+
+def _chart_paths(output: dict | None, parser: LogParser, tools: set[str]) -> list[str]:
+ """How the user saw a chart: ``render-ui``, ``inline`` (the client's UI app), or a ``chart_references`` kind."""
+ paths: list[str] = []
+ calls = parser.get_tool_calls()
+ for call in _render_ui_calls(parser):
+ if not call.is_error and _rendered_tool_name(call) in tools and _render_matches_query(call, calls):
+ paths.append("render-ui")
+ for call in calls:
+ if not call.is_error and call.name in tools and INLINE_VIEW_NOTE in call.output:
+ paths.append("inline")
+ project_ids = frozenset(match for call in calls for match in _PROJECT_URL_RE.findall(call.output))
+ paths.extend(chart_references(assistant_text(output), project_ids, saved_insight_kinds(calls)))
+ return paths
+
+
+def _chart_tools(expected: dict | None, scorer_name: str) -> set[str] | None:
+ spec = _read_spec(expected, scorer_name)
+ if spec is None:
+ return None
+ allowed = spec.get("tool_name_any_of")
+ return {t for t in allowed if isinstance(t, str) and t} if isinstance(allowed, list) else set()
+
+
+class ChartShown(Scorer):
+ """Binary: did the user get the analytics result as a chart?
+
+ ``expected = {"chart_shown": {"tool_name_any_of": ["query-trends"]}}``
+
+ Any chart path counts: ``render-ui`` with a listed query tool, a listed query tool result
+ that the client already displays inline (``INLINE_VIEW_NOTE``), or a full-size chart link or
+ block tag in the answer (a saved insight or a SQL query). ``ChartFromTypedQuery``
+ grades which of these the agent chose. ``None`` if the spec is absent.
+ """
+
+ def _name(self) -> str:
+ return "chart_shown"
+
+ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:
+ tools = _chart_tools(expected, self._name())
+ if tools is None:
+ return Score(name=self._name(), score=None, metadata={"reason": f"No {self._name()} spec on case"})
+ parser = _build_parser(output)
+ if parser is None:
+ return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})
+ paths = _chart_paths(output, parser, tools)
+ if paths:
+ return Score(name=self._name(), score=1.0, metadata={"paths": paths})
+ return Score(
+ name=self._name(),
+ score=0.0,
+ metadata={"reason": "No chart was shown for the expected query tools", "expected_any_of": sorted(tools)},
+ )
+
+
+class ChartFromTypedQuery(Scorer):
+ """Binary: when a chart was shown, did it come from the typed query rather than rewritten SQL?
+
+ ``expected = {"chart_from_typed_query": {"tool_name_any_of": ["query-trends"]}}``
+
+ An agent that answers with ``query-trends`` and then links a hand-written SQL chart shows
+ numbers computed by a different query, without PostHog's standard definitions such as the
+ project timezone or test-account filtering. Score 1.0 when any chart comes from
+ ``render-ui``, the inline view, or a saved insight that holds a typed query; 0.0 when every
+ chart is a SQL query. ``None`` when no chart was shown (graded by ``ChartShown``), when the only
+ other charts are saved insights whose query kind no tool call showed, or the spec is absent.
+ """
+
+ def _name(self) -> str:
+ return "chart_from_typed_query"
+
+ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:
+ tools = _chart_tools(expected, self._name())
+ if tools is None:
+ return Score(name=self._name(), score=None, metadata={"reason": f"No {self._name()} spec on case"})
+ parser = _build_parser(output)
+ if parser is None:
+ return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})
+ paths = _chart_paths(output, parser, tools)
+ if not paths:
+ return Score(name=self._name(), score=None, metadata={"reason": "No chart was shown"})
+ if any(path not in ("sql", "saved") for path in paths):
+ return Score(name=self._name(), score=1.0, metadata={"paths": paths})
+ if "saved" in paths:
+ return Score(
+ name=self._name(),
+ score=None,
+ metadata={"paths": paths, "reason": "No tool call showed the query of the linked saved insight"},
+ )
+ return Score(name=self._name(), score=0.0, metadata={"paths": paths})
+
+
+_TABLE_SEPARATOR_RE = re.compile(r"^(?=.*\|)\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$")
+
+
+def markdown_table_row_counts(text: str) -> list[int]:
+ """Count the data rows of each Markdown table in ``text``, in order."""
+ lines = text.splitlines()
+ counts: list[int] = []
+ index = 1
+ while index < len(lines):
+ if "|" in lines[index - 1] and _TABLE_SEPARATOR_RE.match(lines[index]):
+ rows = 0
+ index += 1
+ while index < len(lines) and "|" in lines[index] and lines[index].strip():
+ rows += 1
+ index += 1
+ counts.append(rows)
+ index += 1
+ return counts
+
+
+class NoMarkdownDataTable(Scorer):
+ """Binary: did the agent avoid restating the query result as a Markdown table?
+
+ ``expected = {"no_markdown_data_table": {}}`` or ``{"no_markdown_data_table": {"max_rows": 3}}``
+
+ A chart already shows the series, so a table that retypes it adds length and hides
+ the conclusion. Small summary tables (at most ``max_rows`` data rows, default 3, such as
+ this period against the previous one) still pass. Score 0.0 when any table in the agent's
+ messages is longer. ``None`` if the spec is absent or the agent wrote no text.
+ """
+
+ DEFAULT_MAX_ROWS = 3
+
+ def _name(self) -> str:
+ return "no_markdown_data_table"
+
+ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:
+ spec = _read_spec(expected, self._name())
+ if spec is None:
+ return Score(name=self._name(), score=None, metadata={"reason": f"No {self._name()} spec on case"})
+
+ text = assistant_text(output)
+ if not text.strip():
+ return Score(name=self._name(), score=None, metadata={"reason": "No agent text"})
+
+ max_rows = spec.get("max_rows", self.DEFAULT_MAX_ROWS)
+ row_counts = markdown_table_row_counts(text)
+ too_long = [rows for rows in row_counts if rows > max_rows]
+ return Score(
+ name=self._name(),
+ score=0.0 if too_long else 1.0,
+ metadata={"table_row_counts": row_counts, "max_rows": max_rows},
+ )
+
+
+class _TableSources(TraversingVisitor):
+ def __init__(self) -> None:
+ self.tables: list[str] = []
+ self.ctes: set[str] = set()
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ self.ctes.update(name.lower() for name in node.ctes or {})
+ super().visit_select_query(node)
+
+ def visit_join_expr(self, node: ast.JoinExpr) -> None:
+ if isinstance(node.table, ast.Field):
+ self.tables.append(".".join(str(part) for part in node.table.chain).lower())
+ super().visit_join_expr(node)
+
+
+def is_catalog_sql(call: ToolCall) -> bool:
+ """True when an ``execute-sql`` call reads only PostHog entity tables (``system.*``).
+
+ Catalog lookups, such as checking for a saved insight or an action, are justified SQL
+ under the routing rules, so they never count as an analytics query. A query that the
+ HogQL parser cannot read counts as analytics SQL.
+ """
+ query = call.input.get("query")
+ if not isinstance(query, str):
+ return False
+ try:
+ sources = _TableSources()
+ sources.visit(parse_select(query))
+ except BaseHogQLError:
+ return False
+ tables = [table for table in sources.tables if table not in sources.ctes]
+ return bool(tables) and all(table.startswith("system.") for table in tables)
diff --git a/products/posthog_ai/evals/product_analytics/eval_chart_not_table.py b/products/posthog_ai/evals/product_analytics/eval_chart_not_table.py
new file mode 100644
index 000000000000..de7efdda29ec
--- /dev/null
+++ b/products/posthog_ai/evals/product_analytics/eval_chart_not_table.py
@@ -0,0 +1,106 @@
+"""Does the agent answer a chart-shaped analytics question with a chart, not a Markdown table?
+
+Agents often run the right query and then retype every data point as a Markdown table, or
+answer with ``execute-sql``, which has no chart at all. Each case asks a question whose
+natural answer is a time series, a breakdown, a funnel, or a retention grid. The agent passes
+when the user sees the result as a chart and the final message states the conclusion
+without a long table.
+
+``ChartShown`` accepts every chart path a harness offers: the MCP client shows the query
+result inline, the agent calls ``render-ui`` with the same query tool and input, or the answer
+holds a full-size chart link or block tag (the PostHog Desktop syntax). ``ChartFromTypedQuery``
+then fails a chart that rewrites the typed result as SQL. PostHog Desktop draws a chart card
+only for a saved insight or a HogQL query, so an agent that does not save an insight can
+reach a chart only through SQL; this scorer measures how often that happens.
+
+The ``table_request_keeps_typed_runner`` control asks for a table explicitly. A table is
+then a correct answer, but the request alone does not justify SQL.
+
+Run on the latest Claude Opus and GPT Sol models, one runtime per run:
+
+ hogli evals eval_chart_not_table --agent-runtime claude --agent-model claude-opus-5-5 --trials 3
+ hogli evals eval_chart_not_table --agent-runtime codex --agent-model gpt-6.1-sol --trials 3
+"""
+
+from __future__ import annotations
+
+from products.posthog_ai.eval_harness.base import SandboxedPublicEval
+from products.posthog_ai.eval_harness.config import SandboxedEvalCase
+from products.posthog_ai.eval_harness.harness.context import EvalContext
+from products.posthog_ai.eval_harness.scorers import NoToolCall
+from products.posthog_ai.evals.cli_mcp.scorers import (
+ ChartFromTypedQuery,
+ ChartShown,
+ DidNotCiteRawSqlAfterTypedQuery,
+ NoMarkdownDataTable,
+)
+from products.posthog_ai.evals.product_analytics.scorers import INSIGHT_WRITE_TOOLS, AnswerFromTypedQuery
+
+
+def _chart_case(name: str, prompt: str, tool: str) -> SandboxedEvalCase:
+ return SandboxedEvalCase(
+ name=name,
+ prompt=prompt,
+ expected={
+ "answer_from_typed_query": {},
+ "chart_shown": {"tool_name_any_of": [tool]},
+ "chart_from_typed_query": {"tool_name_any_of": [tool]},
+ "no_markdown_data_table": {},
+ "did_not_cite_raw_sql_after_typed_query": {"tool": tool},
+ },
+ )
+
+
+async def eval_chart_not_table(ctx: EvalContext) -> None:
+ cases = [
+ _chart_case(
+ "weekly_signups_trend",
+ "How have weekly signups changed over the last 12 weeks?",
+ "query-trends",
+ ),
+ _chart_case(
+ "daily_uploads_by_file_type",
+ "Show daily uploaded_file counts by file_type for the last 30 days.",
+ "query-trends",
+ ),
+ _chart_case(
+ "weekly_file_activity_scorecard",
+ "Give me weekly uploads, downloads, and signups for the last 8 weeks.",
+ "query-trends",
+ ),
+ _chart_case(
+ "pageviews_by_country",
+ "Which countries sent the most pageviews in the last 30 days?",
+ "query-trends",
+ ),
+ _chart_case(
+ "pricing_to_upgrade_funnel",
+ "What does our funnel from a /pricing/ pageview to signed_up to upgraded_plan look like over the last "
+ "30 days?",
+ "query-funnel",
+ ),
+ _chart_case(
+ "weekly_signup_retention",
+ "How well do users who signed up in the last 8 weeks keep coming back each week?",
+ "query-retention",
+ ),
+ SandboxedEvalCase(
+ name="table_request_keeps_typed_runner",
+ prompt="List the weekly signup counts for the last 8 weeks as a table.",
+ expected={"answer_from_typed_query": {}},
+ ),
+ ]
+
+ await SandboxedPublicEval(
+ experiment_name="sandboxed-chart-not-table-cli",
+ cases=cases,
+ scorers=[
+ NoToolCall(forbidden=INSIGHT_WRITE_TOOLS, name="no_persistent_insight_save"),
+ AnswerFromTypedQuery(),
+ ChartShown(),
+ ChartFromTypedQuery(),
+ NoMarkdownDataTable(),
+ DidNotCiteRawSqlAfterTypedQuery(),
+ ],
+ ctx=ctx,
+ )
diff --git a/products/posthog_ai/evals/product_analytics/eval_typed_runner_misses.py b/products/posthog_ai/evals/product_analytics/eval_typed_runner_misses.py
new file mode 100644
index 000000000000..8b52c9c9580e
--- /dev/null
+++ b/products/posthog_ai/evals/product_analytics/eval_typed_runner_misses.py
@@ -0,0 +1,206 @@
+"""Product-analytics requests that agents answer with ``execute-sql`` instead of a typed runner.
+
+Each case reproduces, on Hedgebox, a request shape that the online evaluation
+"MCP SQL: typed query runner choice" fails most often on production MCP traces:
+
+- Several metrics in one question ("counts, uniques and a filtered count"), which agents pack
+ into one SQL query with ``countIf`` and ``uniqIf`` instead of trends series.
+- A count of events that carry a property, a single-day unique-user count, and active users
+ without internal accounts, which agents write as SQL with hand-built filters.
+- Breakdowns, period comparisons, percentiles, and ratios, which trends expresses with a
+ breakdown, ``compareFilter``, property math, and a formula.
+- Funnels and return rates computed with ``uniqIf`` per event, which drop the step order and
+ conversion window that ``query-funnel`` and ``query-retention`` keep.
+- A justified ``system.*`` lookup, followed by an analysis that stays on SQL.
+
+``AnswerFromTypedQuery`` checks which data the answer reports: its values must come from a typed
+query result. A SQL query that checks a typed result is allowed, which is looser than the online
+judge, because the judge fails any SQL call against event data.
+``eval_justified_sql_controls`` holds requests where SQL is the right tool, so a guidance
+change that removes these misses cannot pass by sending everything to typed runners.
+
+Run the suites on the latest Claude Opus and GPT Sol models, one runtime per run, and compare
+the two runs paired by case:
+
+ hogli evals eval_typed_runner_misses --agent-runtime claude --agent-model claude-opus-5-5 --trials 3
+ hogli evals eval_typed_runner_misses --agent-runtime codex --agent-model gpt-6.1-sol --trials 3
+"""
+
+from __future__ import annotations
+
+from typing import Unpack
+
+from products.posthog_ai.eval_harness.base import SandboxedPublicEval
+from products.posthog_ai.eval_harness.config import SandboxedEvalCase
+from products.posthog_ai.eval_harness.harness.context import EvalContext
+from products.posthog_ai.eval_harness.scorers import AnswerToolCallNot, NoToolCall
+from products.posthog_ai.evals.cli_mcp.scorers import FirstRelevantTool
+from products.posthog_ai.evals.product_analytics.scorers import (
+ INSIGHT_WRITE_TOOLS,
+ TYPED_QUERY_TOOLS,
+ AnswerFromTypedQuery,
+ InsightShape,
+ InsightShapeSpec,
+)
+
+from .eval_sql_shaped_routing import ANALYSIS_QUERY_TOOLS
+
+
+def _case(
+ name: str, prompt: str, *, first_tool: str | None = None, **shape: Unpack[InsightShapeSpec]
+) -> SandboxedEvalCase:
+ expected: dict[str, object] = {}
+ if first_tool is None:
+ expected["answer_from_typed_query"] = {}
+ else:
+ expected["first_relevant_tool"] = {"tool": first_tool}
+ if shape:
+ expected["insight_shape"] = shape
+ return SandboxedEvalCase(name=name, prompt=prompt, expected=expected)
+
+
+async def eval_typed_runner_misses(ctx: EvalContext) -> None:
+ cases = [
+ _case(
+ "scorecard_counts_and_uploaders",
+ "For the last 7 days, give me the number of uploaded_file events, the number of downloaded_file events, "
+ "and how many unique users uploaded a file.",
+ tool="query-trends",
+ events=["uploaded_file", "downloaded_file", "uploaded_file"],
+ math=["total", "dau"],
+ ),
+ _case(
+ "count_with_file_name_present",
+ "How many uploaded_file events were there in the last 30 days, and how many of them had a file_name?",
+ tool="query-trends",
+ events=["uploaded_file", "uploaded_file"],
+ filtered_on=["file_name"],
+ ),
+ _case(
+ "unique_signups_yesterday",
+ "How many unique users signed up yesterday?",
+ tool="query-trends",
+ events=["signed_up"],
+ math=["dau"],
+ ),
+ _case(
+ "active_users_excluding_internal",
+ "What were our weekly and monthly active users over the last 30 days, excluding internal and test "
+ "accounts? Count anyone who fired logged_in.",
+ tool="query-trends",
+ math=["weekly_active", "monthly_active"],
+ filter_test_accounts=True,
+ ),
+ _case(
+ "downloads_by_file_type_counts_and_uniques",
+ "Break down yesterday's downloaded_file events by file_type, with the event count and the number of "
+ "unique users for each type.",
+ tool="query-trends",
+ breakdown="file_type",
+ math=["total", "dau"],
+ ),
+ _case(
+ "uploads_by_os_vs_previous_week",
+ "Compare uploaded_file events per operating system over the last 7 days with the 7 days before.",
+ tool="query-trends",
+ breakdown="$os",
+ compare=True,
+ events=["uploaded_file"],
+ ),
+ _case(
+ "net_plan_upgrades_formula",
+ "What's the net number of plan upgrades minus downgrades per week over the last 8 weeks?",
+ tool="query-trends",
+ events=["upgraded_plan", "downgraded_plan"],
+ formula=True,
+ ),
+ _case(
+ "file_size_percentiles_vs_previous_period",
+ "What are the median and 95th percentile file_size_b of uploaded files by file_type over the last 7 "
+ "days, compared with the 7 days before?",
+ tool="query-trends",
+ breakdown="file_type",
+ compare=True,
+ math=["median", "p95"],
+ ),
+ _case(
+ "pricing_session_share_weekly",
+ "What share of weekly sessions with a pageview included a pageview of /pricing/, over the last 8 weeks?",
+ tool="query-trends",
+ events=["$pageview", "$pageview"],
+ math=["unique_session"],
+ formula=True,
+ filtered_on=["$pathname", "$current_url"],
+ ),
+ _case(
+ "dashboard_tile_signup_funnel",
+ "I'm preparing an operations dashboard tile. Compute the 30-day signup funnel: a pageview of /signup/, "
+ "then signed_up, then uploaded_file, then upgraded_plan. Don't save anything yet.",
+ tool="query-funnel",
+ event_sequence=["$pageview", "signed_up", "uploaded_file", "upgraded_plan"],
+ ),
+ _case(
+ "signup_to_upload_last_14_days_vs_prior",
+ "Did conversion from signed_up to uploaded_file change in the last 14 days compared with the 14 days "
+ "before?",
+ tool="query-funnel",
+ event_sequence=["signed_up", "uploaded_file"],
+ ),
+ _case(
+ "day1_day7_return_rates",
+ "For users who first fired logged_in in the last 30 days, what are their day-1 and day-7 return rates?",
+ tool="query-retention",
+ ),
+ # The system.* lookup is justified SQL and does not count as analytics SQL.
+ _case(
+ "existing_insight_check_then_query",
+ "Check whether we already have a saved insight for weekly logins by browser. If we don't, show me "
+ "weekly logged_in counts by $browser over the last 8 weeks.",
+ tool="query-trends",
+ breakdown="$browser",
+ events=["logged_in"],
+ ),
+ ]
+
+ await SandboxedPublicEval(
+ experiment_name="sandboxed-typed-runner-misses-cli",
+ cases=cases,
+ scorers=[
+ NoToolCall(forbidden=INSIGHT_WRITE_TOOLS, name="no_persistent_insight_save"),
+ AnswerFromTypedQuery(),
+ AnswerToolCallNot("execute-sql", TYPED_QUERY_TOOLS),
+ InsightShape(),
+ ],
+ ctx=ctx,
+ )
+
+
+async def eval_justified_sql_controls(ctx: EvalContext) -> None:
+ cases = [
+ _case(
+ "sql_explicit_hogql_request",
+ "Write and run a HogQL query that counts uploaded_file events per day over the last 14 days.",
+ first_tool="execute-sql",
+ ),
+ _case(
+ "sql_latest_event_timestamp",
+ "When did the most recent uploaded_file event arrive? Give me its exact timestamp.",
+ first_tool="execute-sql",
+ ),
+ _case(
+ "sql_gap_between_first_and_second_upload",
+ "For each user who uploaded a file in the last 30 days, compute the hours between their first and "
+ "second uploaded_file event, and show the 10 users with the longest gap.",
+ first_tool="execute-sql",
+ ),
+ ]
+
+ await SandboxedPublicEval(
+ experiment_name="sandboxed-justified-sql-controls-cli",
+ cases=cases,
+ scorers=[
+ NoToolCall(forbidden=INSIGHT_WRITE_TOOLS, name="no_persistent_insight_save"),
+ FirstRelevantTool(relevant_tools=ANALYSIS_QUERY_TOOLS),
+ ],
+ ctx=ctx,
+ )
diff --git a/products/posthog_ai/evals/product_analytics/scorers.py b/products/posthog_ai/evals/product_analytics/scorers.py
index e19b3615f64c..2f70f67d1460 100644
--- a/products/posthog_ai/evals/product_analytics/scorers.py
+++ b/products/posthog_ai/evals/product_analytics/scorers.py
@@ -13,8 +13,11 @@ and only then runs the actual query tool.
from __future__ import annotations
+import re
+import json
from collections.abc import Iterable
-from typing import Any
+from decimal import Decimal, InvalidOperation
+from typing import Any, TypedDict
from pydantic import BaseModel
@@ -29,6 +32,7 @@ from products.posthog_ai.eval_harness.scorers import (
JudgedScorer,
)
from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer
+from products.posthog_ai.evals.cli_mcp.scorers import assistant_text, is_catalog_sql, tool_output_text
QUERY_TRENDS_TOOL_NAME = "query-trends"
QUERY_RETENTION_TOOL_NAME = "query-retention"
@@ -732,6 +736,22 @@ _ANSWER_TOOL_SCHEMAS: dict[str, type[BaseModel]] = {
}
+class InsightShapeSpec(TypedDict, total=False):
+ """The keys ``InsightShape`` reads from ``expected["insight_shape"]``."""
+
+ tool: str | list[str]
+ display: str | list[str]
+ compare: bool
+ funnel_viz: str | list[str]
+ breakdown: str | None
+ events: list[str | int]
+ event_sequence: list[str | int]
+ math: list[str]
+ formula: bool
+ filter_test_accounts: bool
+ filtered_on: list[str]
+
+
class InsightShape(Scorer):
"""Binary: did the agent's answer query have the expected skeleton?
@@ -751,9 +771,14 @@ class InsightShape(Scorer):
- ``breakdown``: the breakdown property name, or ``None`` to require no breakdown.
- ``events``: the multiset of series events, in any order.
- ``event_sequence``: the series events in order (funnel steps).
+ - ``math``: math types that must each appear on at least one series; an omitted
+ math counts as ``total``.
+ - ``formula``: whether a trends query combines its series with a formula.
+ - ``filter_test_accounts``: whether the query excludes internal and test accounts.
+ - ``filtered_on``: property keys of which at least one must appear in a query or series filter.
The keys after ``tool`` describe a series query, so they are checked only when
- the chosen tool is ``query-trends`` or ``query-funnel``. Filters, math, and date
+ the chosen tool is ``query-trends`` or ``query-funnel``. Property filters and date
ranges are left to the schema-alignment judges.
"""
@@ -793,6 +818,10 @@ class InsightShape(Scorer):
"funnel_viz": _actual_funnel_viz(query, tool_name),
"breakdown": _actual_breakdown(query),
"event_sequence": _actual_events(query),
+ "math": _actual_math(query),
+ "formula": _actual_formula(query, tool_name),
+ "filter_test_accounts": query.get("filterTestAccounts") is True,
+ "filtered_on": sorted(_filtered_properties(query)),
}
actual["events"] = sorted(actual["event_sequence"], key=str)
checked = shape if tool_name in SERIES_QUERY_TOOLS else {"tool": shape.get("tool")}
@@ -835,8 +864,12 @@ def _shape_field_matches(key: str, expected_value: Any, actual: dict[str, Any])
if key in ("tool", "display", "funnel_viz"):
accepted = expected_value if isinstance(expected_value, list | tuple | set) else [expected_value]
return actual[key] in {str(item) for item in accepted}
- if key in ("compare", "breakdown"):
+ if key in ("compare", "breakdown", "formula", "filter_test_accounts"):
return actual[key] == expected_value
+ if key == "filtered_on":
+ return isinstance(expected_value, list) and bool(set(expected_value) & set(actual["filtered_on"]))
+ if key == "math":
+ return isinstance(expected_value, list) and set(expected_value) <= set(actual["math"])
if key == "events":
return isinstance(expected_value, list) and sorted(expected_value, key=str) == actual["events"]
if key == "event_sequence":
@@ -882,17 +915,159 @@ def _actual_breakdown(query: dict[str, Any]) -> str | None:
return str(breakdown) if breakdown else None
-def _actual_events(query: dict[str, Any]) -> list[str | float | None]:
- events: list[str | float | None] = []
+def _series_nodes(query: dict[str, Any]) -> list[dict[str, Any]]:
+ nodes: list[dict[str, Any]] = []
for node in query.get("series") or []:
if not isinstance(node, dict):
continue
inner = node.get("nodes") if node.get("kind") == "GroupNode" else [node]
- for entry in inner or []:
- if not isinstance(entry, dict):
- continue
- if "event" in entry:
- events.append(entry.get("event"))
- elif entry.get("kind") == "ActionsNode":
- events.append(entry.get("id"))
+ nodes.extend(entry for entry in inner or [] if isinstance(entry, dict))
+ return nodes
+
+
+def _actual_math(query: dict[str, Any]) -> list[str]:
+ # A GroupNode's own math applies to the whole group; the engine ignores the math of its inner nodes.
+ series = [node for node in query.get("series") or [] if isinstance(node, dict)]
+ return sorted({str(node.get("math") or "total") for node in series})
+
+
+def _filtered_properties(query: dict[str, Any]) -> set[str]:
+ """Property keys used in the query's own filters or in any series filter."""
+ keys: set[str] = set()
+
+ def collect(filters: Any) -> None:
+ if isinstance(filters, dict):
+ if isinstance(filters.get("key"), str):
+ keys.add(filters["key"])
+ collect(filters.get("values"))
+ elif isinstance(filters, list):
+ for item in filters:
+ collect(item)
+
+ collect(query.get("properties"))
+ for node in query.get("series") or []:
+ if isinstance(node, dict):
+ collect(node.get("properties"))
+ for node in _series_nodes(query):
+ collect(node.get("properties"))
+ return keys
+
+
+def _actual_formula(query: dict[str, Any], tool_name: str) -> bool | None:
+ if tool_name != QUERY_TRENDS_TOOL_NAME:
+ return None
+ trends_filter = query.get("trendsFilter")
+ if not isinstance(trends_filter, dict):
+ return False
+ # The assistant schema uses ``formulaNodes``; saved TrendsFilter queries can carry the older keys.
+ return any(trends_filter.get(key) for key in ("formulaNodes", "formulas", "formula"))
+
+
+def _actual_events(query: dict[str, Any]) -> list[str | float | None]:
+ events: list[str | float | None] = []
+ for entry in _series_nodes(query):
+ if "event" in entry:
+ events.append(entry.get("event"))
+ elif entry.get("kind") == "ActionsNode":
+ events.append(entry.get("id"))
return events
+
+
+# Text that holds digits but no reported values: links (URL encoding such as ``%20``), ISO dates and
+# times, clock times, and "Oct 12" style dates.
+_NON_VALUE_RES = (
+ re.compile(r"https?://\S+"),
+ re.compile(r"\d{4}-\d{2}-\d{2}(?:[T ][\d:.]+)?(?:Z|[+-]\d{2}:?\d{2})?"),
+ re.compile(r"\b\d{1,2}:\d{2}(?::\d{2})?\b"),
+ re.compile(r"\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}\b", re.IGNORECASE),
+ re.compile(r"\b\d{1,2}\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\b", re.IGNORECASE),
+)
+# A minus sign counts only at the start of a value, so the range "6-16" reads as 6 and 16, not 6 and -16.
+_VALUE_RE = re.compile(r"(?<![\w.])[-\u2212]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?(?![\w])")
+
+
+def reported_values(text: str, ignore: frozenset[str] = frozenset(), *, skip_years: bool = True) -> set[str]:
+ """Data values in ``text``, normalized so ``1,234``, ``1234`` and ``1234.0`` compare equal.
+
+ Single-digit integers and years written without a thousands separator are dropped, because they
+ also appear as window sizes, step numbers, and dates, so they cannot show where a value came from.
+ An answer whose only values are of this kind gets no score rather than a wrong one. Result tables
+ pass ``skip_years=False``, because a year-like value there is a data cell.
+ """
+ for pattern in _NON_VALUE_RES:
+ text = pattern.sub(" ", text)
+ values: set[str] = set()
+ for raw in _VALUE_RE.findall(text):
+ try:
+ number = Decimal(raw.replace(",", "").replace("\u2212", "-"))
+ except InvalidOperation:
+ continue
+ if number == number.to_integral_value() and (
+ abs(number) < 10 or (skip_years and "," not in raw and 1900 <= number <= 2100)
+ ):
+ continue
+ value = format(number.normalize(), "f")
+ if value not in ignore:
+ values.add(value)
+ return values
+
+
+def _result_text(call: ToolCall) -> str:
+ """The result rows of a query tool call, without the query, notes, and examples the tool adds around them."""
+ text = tool_output_text(call.output)
+ try:
+ payload = json.loads(text)
+ except (TypeError, ValueError):
+ payload = None
+ if isinstance(payload, dict) and "results" in payload:
+ return json.dumps(payload["results"])
+ rows = [line for line in text.splitlines() if "|" in line]
+ return "\n".join(rows) if rows else text
+
+
+class AnswerFromTypedQuery(Scorer):
+ """Binary: does the answer report data from a typed query result rather than from SQL?
+
+ ``expected = {"answer_from_typed_query": {}}``
+
+ Compares the values in the agent's messages with the values each tool returned. The case
+ passes when the answer reports at least one value from a successful typed query result. A
+ SQL query that checks a typed result is allowed, and the answer can mention its values too.
+ The case fails when the reported values come only from analytics SQL results, or when
+ analytics SQL ran and no typed query did. ``system.*`` catalog lookups are not analytics SQL.
+ ``None`` when the answer has no value traceable to either source, or the spec is absent.
+ """
+
+ def _name(self) -> str:
+ return "answer_from_typed_query"
+
+ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:
+ if not isinstance(expected, dict) or not isinstance(expected.get(self._name()), dict):
+ return Score(name=self._name(), score=None, metadata={"reason": f"No {self._name()} key on case"})
+ parser = parser_for(output)
+ if parser is None:
+ return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})
+
+ calls = [call for call in parser.get_tool_calls() if not call.is_error]
+ typed_calls = [call for call in calls if call.name in TYPED_QUERY_TOOLS]
+ sql_calls = [call for call in calls if call.name == "execute-sql" and not is_catalog_sql(call)]
+ if not typed_calls:
+ if sql_calls:
+ return Score(name=self._name(), score=0.0, metadata={"reason": "Only analytics SQL ran"})
+ return Score(name=self._name(), score=None, metadata={"reason": "No query ran"})
+
+ ignore = frozenset(reported_values(user_prompt(output)))
+ answer = reported_values(assistant_text(output), ignore)
+ typed = set().union(*(reported_values(_result_text(call), skip_years=False) for call in typed_calls))
+ sql = set().union(*(reported_values(_result_text(call), skip_years=False) for call in sql_calls))
+ sql_only = sql - typed
+ typed_hits = sorted(answer & typed)
+ sql_hits = sorted(answer & sql_only)
+ metadata = {"typed_values_reported": typed_hits, "sql_only_values_reported": sql_hits}
+ if typed_hits:
+ return Score(name=self._name(), score=1.0, metadata=metadata)
+ if sql_hits:
+ return Score(name=self._name(), score=0.0, metadata=metadata)
+ return Score(
+ name=self._name(), score=None, metadata={**metadata, "reason": "No reported value matches a query result"}
+ )