:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(posthog-ai): add evals for sql routing misses and chart answers (#119445)
Daniel Parkerf8c9518eb20e
diff --git a/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py b/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.pyindex d32f20aaafcd..de14426be006 100644--- a/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py+++ b/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py@@ -1,10 +1,19 @@from __future__ import annotationsimport json+from urllib.parse import quotefrom parameterized import parameterized-from products.posthog_ai.evals.cli_mcp.scorers import DidNotCiteRawSqlAfterTypedQuery, FirstRelevantTool+from products.posthog_ai.eval_harness.test.test_eval_scorers import _raw_tool_log+from products.posthog_ai.evals.cli_mcp.scorers import (+ INLINE_VIEW_NOTE,+ ChartFromTypedQuery,+ ChartShown,+ DidNotCiteRawSqlAfterTypedQuery,+ FirstRelevantTool,+ NoMarkdownDataTable,+)def _tool_call_start(call_id: str, command: str) -> str:@@ -241,3 +250,153 @@ def test_first_relevant_tool_turn_skips_a_missing_slice() -> None:assert result.score is Noneassert "No turn 9 log slice" in result.metadata["reason"]+++_TRENDS_RESULT: tuple[str, dict[str, list[str]], str] = ("mcp__posthog__query-trends", {"series": []}, "rows")+_TRENDS_INLINE_RESULT: tuple[str, dict[str, list[str]], str] = (+ "mcp__posthog__query-trends",+ {"series": []},+ f"rows\n\nThe user {INLINE_VIEW_NOTE}.",+)+_BASE = "https://us.example.com/project/1"+_SQL_CHART = f"[Weekly signups]({_BASE}/sql?open_query=SELECT%20toStartOfWeek(timestamp)%2C%20count()%20FROM%20events)"+_INSIGHT_CHART = f"[Weekly signups]({_BASE}/insights/9pQx3)"+_HOGQL_NODE = '{"kind": "DataTableNode", "source": {"kind": "HogQLQuery", "query": "SELECT 1"}}'+_TRENDS_NODE = '{"kind": "InsightVizNode", "source": {"kind": "TrendsQuery", "series": []}}'+_PROJECT_RESULT: tuple[str, dict[str, list[str]], str] = (+ "mcp__posthog__query-trends",+ {"series": []},+ f"rows\n{_BASE}/insights/new",+)+_UNSAVED_INSIGHT_LINK = f"[Weekly signups]({_BASE}/insights/new#q=%7B%7D)"+_SAVED_TRENDS: tuple[str, dict[str, str], str] = (+ "mcp__posthog__insight-get",+ {"id": "9pQx3"},+ '{"short_id": "9pQx3", "query": {"kind": "InsightVizNode", "source": {"kind": "TrendsQuery"}}}',+)+_SAVED_HOGQL: tuple[str, dict[str, str], str] = (+ "mcp__posthog__insight-get",+ {"id": "9pQx3"},+ "short_id: 9pQx3\nquery:\n kind: DataVisualizationNode\n source:\n kind: HogQLQuery",+)+++def _render(tool_name: str, status: str = "completed", tool_input: dict | None = None) -> tuple:+ rendered = {"series": []} if tool_input is None else tool_input+ return ("mcp__posthog__render-ui", {"tool_name": tool_name, "tool_input": rendered}, "Rendering", status)+++def _chart_output(calls: list[tuple], answer: str = "Signups rose.") -> dict:+ return {+ "raw_log": _raw_tool_log(calls),+ "prompt": "Weekly signups",+ "messages": [{"role": "assistant", "content": [{"type": "text", "text": answer}]}],+ }+++ [+ ("rendered_query_tool", [_TRENDS_RESULT, _render("query-trends")], "Signups rose.", 1.0, 1.0),+ ("inline_view_note", [_TRENDS_INLINE_RESULT], "Signups rose.", 1.0, 1.0),+ (+ "saved_insight_link",+ [_TRENDS_RESULT, _SAVED_TRENDS],+ f"Signups rose.\n\n{_INSIGHT_CHART}\n\nMore text.",+ 1.0,+ 1.0,+ ),+ ("saved_hogql_insight_link", [_TRENDS_RESULT, _SAVED_HOGQL], f"Signups rose.\n\n{_INSIGHT_CHART}", 1.0, 0.0),+ ("saved_insight_of_unknown_kind", [_TRENDS_RESULT], f"Signups rose.\n\n{_INSIGHT_CHART}", 1.0, None),+ ("sql_link_after_typed_query", [_TRENDS_RESULT], f"Signups rose.\n\n{_SQL_CHART}", 1.0, 0.0),+ ("sql_link_inside_sentence", [_TRENDS_RESULT], f"See {_SQL_CHART} for details.", 0.0, None),+ ("unsaved_insight_link", [_TRENDS_RESULT], f"Signups rose.\n\n{_UNSAVED_INSIGHT_LINK}", 0.0, None),+ (+ "unsaved_trends_link",+ [_TRENDS_RESULT],+ f"Up.\n\n[Signups]({_BASE}/insights/new#q={quote(_TRENDS_NODE)})",+ 0.0,+ None,+ ),+ (+ "unsaved_hogql_link",+ [_TRENDS_RESULT],+ f"Up.\n\n[Signups]({_BASE}/insights/new#q={quote(_HOGQL_NODE)})",+ 1.0,+ 0.0,+ ),+ ("short_insight_link", [_TRENDS_RESULT, _SAVED_TRENDS], f"Up.\n\n[Signups]({_BASE}/i/9pQx3)", 1.0, 1.0),+ ("same_project_link", [_PROJECT_RESULT, _SAVED_TRENDS], f"Up.\n\n{_INSIGHT_CHART}", 1.0, 1.0),+ (+ "other_project_link",+ [_PROJECT_RESULT],+ "Up.\n\n[Signups](https://us.example.com/project/2/insights/9pQx3)",+ 0.0,+ None,+ ),+ ("failed_render", [_TRENDS_RESULT, _render("query-trends", status="failed")], "Signups rose.", 0.0, None),+ ("rendered_other_tool", [_TRENDS_RESULT, _render("experiment-list")], "Signups rose.", 0.0, None),+ (+ "render_with_empty_input",+ [_TRENDS_RESULT, _render("query-trends", tool_input={})],+ "Signups rose.",+ 0.0,+ None,+ ),+ (+ "render_with_other_input",+ [_TRENDS_RESULT, _render("query-trends", tool_input={"series": [{"event": "x"}]})],+ "Signups rose.",+ 0.0,+ None,+ ),+ (+ "block_tag_alone",+ [_TRENDS_RESULT, _SAVED_TRENDS],+ 'Signups rose.\n\n<insight id="9pQx3" display="block"/>',+ 1.0,+ 1.0,+ ),+ ("block_tag_in_sentence", [_TRENDS_RESULT], 'See <insight id="9pQx3" display="block"/> here.', 0.0, None),+ ("block_tag_without_id", [_TRENDS_RESULT], 'Signups rose.\n\n<insight display="block"/>', 0.0, None),+ (+ "block_tag_in_code_fence",+ [_TRENDS_RESULT],+ 'Use this tag:\n\n```\n<insight id="9pQx3" display="block"/>\n```',+ 0.0,+ None,+ ),+ ]+)+def test_chart_scorers_grade_each_chart_path(+ _name: str, calls: list[tuple], answer: str, shown: float, typed: float | None+) -> None:+ output = _chart_output(calls, answer)+ spec = {"tool_name_any_of": ["query-trends"]}++ assert ChartShown()._run_eval_sync(output, expected={"chart_shown": spec}).score == shown+ assert ChartFromTypedQuery()._run_eval_sync(output, expected={"chart_from_typed_query": spec}).score == typed+++_LONG_TABLE = "| Week | Signups |\n|---|---|\n| W1 | 10 |\n| W2 | 12 |\n| W3 | 9 |\n| W4 | 14 |"+_SUMMARY_TABLE = "| Period | Signups |\n| :--- | ---: |\n| Last 7 days | 45 |\n| Previous 7 days | 38 |"+++ [+ ("long_table", [f"Signups rose.\n\n{_LONG_TABLE}\n\nThey peaked in W4."], 0.0),+ ("single_column_table", ["Weekly signups:\n\n| Signups |\n| --- |\n| 10 |\n| 12 |\n| 9 |\n| 14 |"], 0.0),+ ("dashes_under_piped_line", ["a | b\n---\nSignups rose."], 1.0),+ ("long_table_short_delimiters", [f"Signups rose.\n\n{_LONG_TABLE.replace('|---|---|', '| :- | -: |')}"], 0.0),+ ("long_table_before_closing_line", [f"Signups rose.\n\n{_LONG_TABLE}", "I added two buttons above."], 0.0),+ ("summary_table", [f"Signups rose 18%.\n\n{_SUMMARY_TABLE}"], 1.0),+ ("prose_with_pipes", ["Signups rose 18% | weekly peak in W4 | no drop."], 1.0),+ ]+)+def test_no_markdown_data_table_flags_long_tables(_name: str, texts: list[str], score: float) -> None:+ messages = [{"role": "assistant", "content": [{"type": "text", "text": text}]} for text in texts]++ result = NoMarkdownDataTable()._run_eval_sync(+ {"messages": messages, "last_message": texts[-1]}, expected={"no_markdown_data_table": {}}+ )++ assert result.score == scorediff --git a/products/posthog_ai/eval_harness/test/test_visualization_choice_scorers.py b/products/posthog_ai/eval_harness/test/test_visualization_choice_scorers.pyindex 87a33af9225a..1f7bfa798508 100644--- a/products/posthog_ai/eval_harness/test/test_visualization_choice_scorers.py+++ b/products/posthog_ai/eval_harness/test/test_visualization_choice_scorers.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport pytestfrom products.posthog_ai.eval_harness.test.test_eval_scorers import _raw_tool_log-from products.posthog_ai.evals.product_analytics.scorers import InsightShape+from products.posthog_ai.evals.product_analytics.scorers import AnswerFromTypedQuery, InsightShapedef _output(calls) -> dict:@@ -54,6 +54,30 @@ UPLOADS_BY_FILE_TYPE = {}+def _scorecard_call(maths: list[str | None], formula: bool = False, test_accounts: bool | None = None) -> tuple:+ series = [{"kind": "EventsNode", "event": "uploaded_file", **({"math": m} if m else {})} for m in maths]+ query: dict = {"series": series}+ if formula:+ query["trendsFilter"] = {"formulaNodes": [{"formula": "A-B"}]}+ if test_accounts is not None:+ query["filterTestAccounts"] = test_accounts+ return ("mcp__posthog__query-trends", query, "ok")+++_NODES = [{"kind": "EventsNode", "event": "uploaded_file"}, {"kind": "EventsNode", "event": "downloaded_file"}]+_DAU_NODES = [{**node, "math": "dau"} for node in _NODES]+++def _filtered_call(series_key: str | None = None, query_key: str | None = None) -> tuple:+ node: dict = {"kind": "EventsNode", "event": "uploaded_file"}+ if series_key:+ node["properties"] = [{"key": series_key, "operator": "is_set", "type": "event"}]+ query: dict = {"series": [node]}+ if query_key:+ query["properties"] = {"type": "AND", "values": [{"key": query_key, "value": "/pricing/", "type": "event"}]}+ return ("mcp__posthog__query-trends", query, "ok")++def _sql_call(status: str = "completed") -> tuple:return ("mcp__posthog__execute-sql", {"query": "SELECT 1"}, "1", status)@@ -209,6 +233,28 @@ PAGEVIEWS_BY_BROWSER = {0.0,None,),+ ([_scorecard_call(["dau", None])], {"math": ["total", "dau"]}, 1.0, []),+ ([_scorecard_call([None, None])], {"math": ["total", "dau"]}, 0.0, ["math"]),+ ([_scorecard_call(["median", "p95"])], {"math": ["p95"]}, 1.0, []),+ ([_scorecard_call(["dau", None], formula=True)], {"formula": True}, 1.0, []),+ ([_scorecard_call(["dau", None])], {"formula": True}, 0.0, ["formula"]),+ ([_scorecard_call(["dau"], test_accounts=True)], {"filter_test_accounts": True}, 1.0, []),+ ([_scorecard_call(["dau"])], {"filter_test_accounts": True}, 0.0, ["filter_test_accounts"]),+ (+ [("mcp__posthog__query-trends", {"series": [{"kind": "GroupNode", "math": "dau", "nodes": _NODES}]}, "ok")],+ {"math": ["dau"]},+ 1.0,+ [],+ ),+ (+ [("mcp__posthog__query-trends", {"series": [{"kind": "GroupNode", "nodes": _DAU_NODES}]}, "ok")],+ {"math": ["dau"]},+ 0.0,+ ["math"],+ ),+ ([_filtered_call(series_key="file_name")], {"filtered_on": ["file_name"]}, 1.0, []),+ ([_filtered_call(query_key="$pathname")], {"filtered_on": ["$pathname", "$current_url"]}, 1.0, []),+ ([_filtered_call()], {"filtered_on": ["file_name"]}, 0.0, ["filtered_on"]),],)def test_insight_shape_checks_only_the_expected_keys(calls, shape, score, mismatched) -> None:@@ -220,3 +266,69 @@ def test_insight_shape_checks_only_the_expected_keys(calls, shape, score, mismatdef test_insight_shape_skips_without_expectation() -> None:assert InsightShape()._run_eval_sync(_output([_trends_call()]), {}).score is None+++def _result(tool: str, text: str, query: dict | None = None) -> tuple:+ return (f"mcp__posthog__{tool}", query or {}, {"content": [{"type": "text", "text": text}]})+++_TYPED = _result("query-trends", "Date range|Uploads|Unique uploaders\n2026-10-12 to 2026-10-19|21|917")+_SQL = _result("execute-sql", "uploads|downloads\n20|846", {"query": "SELECT count() FROM events"})+_CATALOG = _result("execute-sql", "id|name\n42|Weekly logins", {"query": "SELECT id, name FROM system.insights"})+_SQL_LINK = "[Uploads](https://us.example.com/project/1/sql?open_query=SELECT%20count()%20FROM%20events)"+++ "calls,answer,score",+ [+ ([_TYPED], "Oct 12 to Oct 19: 21 uploads and 917 unique uploaders.", 1.0),+ (+ [_TYPED, _SQL],+ f"There were 21 uploads and 917 downloads.\n\n{_SQL_LINK}\n\nA rolling window gives 20 and 846.",+ 1.0,+ ),+ ([_TYPED, _SQL], "There were 20 uploads and 846 downloads.", 0.0),+ ([_SQL], "There were 20 uploads and 846 downloads.", 0.0),+ ([_CATALOG, _TYPED], "No saved insight exists. There were 21 uploads.", 1.0),+ ([_CATALOG], "Insight 42 already tracks weekly logins.", None),+ ([_result("execute-sql", "n\n846", {"query": "SELECT count() FROM system.insights, events"})], "846", 0.0),+ (+ [_result("execute-sql", "n\n846", {"query": "SELECT count() FROM `events` JOIN system.insights i ON 1"})],+ "846",+ 0.0,+ ),+ ([_TYPED, _SQL], "Uploads rose by a third.", None),+ ([_result("query-trends", "Net|-42"), _result("execute-sql", "net\n42")], "Net upgrades: 42.", 0.0),+ ([_result("query-trends", "Net|-42"), _result("execute-sql", "net\n42")], "Net upgrades: -42.", 1.0),+ (+ [+ _result("query-funnel", "Step|Conversion\nUploaded file|45\n\nExample: 90% of users convert."),+ _result("execute-sql", "rate\n90"),+ ],+ "Conversion is 90%.",+ 0.0,+ ),+ ([_result("query-trends", "Signups|2026")], "There were 2,026 signups.", 1.0),+ (+ [+ _result(+ "query-trends",+ '{"query": {"trendsFilter": {"smoothingIntervals": 30}}, "results": [{"label": "x", "count": 917}]}',+ ),+ _result("execute-sql", "users\n30"),+ ],+ "30 users uploaded a file.",+ 0.0,+ ),+ ],+)+def test_answer_from_typed_query_traces_reported_values(calls, answer, score) -> None:+ output = {+ "raw_log": _raw_tool_log(calls),+ "prompt": "How many uploads in the last 7 days?",+ "messages": [{"role": "assistant", "content": [{"type": "text", "text": answer}]}],+ }++ result = AnswerFromTypedQuery()._run_eval_sync(output, {"answer_from_typed_query": {}})++ assert result.score == scorediff --git a/products/posthog_ai/evals/cli_mcp/scorers.py b/products/posthog_ai/evals/cli_mcp/scorers.pyindex 30b6e2876e65..6e57f9e10b3a 100644--- a/products/posthog_ai/evals/cli_mcp/scorers.py+++ b/products/posthog_ai/evals/cli_mcp/scorers.py@@ -31,11 +31,24 @@ unrelated cases don't drag the rollup down.before any successful ``query-*`` call?* ``DidNotCiteRawSqlAfterTypedQuery`` — after a typed query tool succeeded,did the final answer avoid hand-writing the equivalent raw SQL?+* ``ChartShown`` — did the user see the query result as a chart, through+ ``render-ui``, the client's inline view, or a chart link in the answer?+* ``ChartFromTypedQuery`` — was that chart the typed query, not rewritten SQL?+* ``NoMarkdownDataTable`` — did the agent avoid retyping the result as a long+ Markdown table?"""from __future__ import annotationsimport re+import json+from collections.abc import Mapping+from urllib.parse import parse_qs, urlsplit++from posthog.hogql import ast+from posthog.hogql.errors import BaseHogQLError+from posthog.hogql.parser import parse_select+from posthog.hogql.visitor import TraversingVisitorfrom products.posthog_ai.backend.exec_commands import INFO_SYNTHETIC_PREFIXfrom products.posthog_ai.eval_harness.log_parser import EXEC_TOOL_NAME, LogParser, ToolCall, is_schema_discovery_call@@ -43,12 +56,15 @@ from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer__all__ = ["CalledTargetTool",+ "ChartFromTypedQuery",+ "ChartShown","DidNotCiteRawSqlAfterTypedQuery","DidNotRenderUi","DrilledIntoSchema","ExecBeforeRender","InfoBeforeCall","FirstRelevantTool",+ "NoMarkdownDataTable","PreferredSearchOverTools","RanPythonPostProcessing","RecoveredToCorrectTool",@@ -1088,3 +1104,407 @@ class DidNotRenderUi(Scorer):score=1.0,metadata={"scope": "any render-ui" if target_set is None else sorted(target_set)},)+++# Mirrors ``UI_APP_RENDER_NOTE`` in ``services/mcp/src/lib/build-tool-result.ts``: the MCP server appends it to a+# query result when the client already displays that result's UI app inline.+INLINE_VIEW_NOTE = "already sees this result as an interactive view"++# A Markdown link alone in its paragraph, which PostHog Desktop draws as a full-size chart card.+# The URL runs to the closing parenthesis because SQL in ``open_query`` keeps unencoded ``(`` and ``)``.+_CHART_LINK_RE = re.compile(r'^\[[^\]]+\]\((https?://\S+?)(?:\s+"[^"]*")?\)$')+# A block-display object tag alone in its paragraph, the other chart syntax in+# ``products/desktop/packages/ui/src/utils/remarkObjectTags.ts``. A tag inside a sentence renders as a chip.+_CHART_TAG_RE = re.compile(r"^<(insight|hogql)\b([^>]*?)/?>(.*?)(?:</\1>)?$", re.IGNORECASE | re.DOTALL)+_TAG_ATTR_RE = re.compile(r'(\w+)="([^"]*)"')+_CODE_FENCE_RE = re.compile(r"^(```|~~~).*?^\1[^\n]*$", re.MULTILINE | re.DOTALL)+_PROJECT_URL_RE = re.compile(r"/project/(\d+)/")+++def assistant_text(output: dict | None) -> str:+ """All text the agent wrote to the user, across every assistant message.++ ``last_message`` is only the final text block, which is often a closing line written after+ a follow-up tool call, so presentation checks read the whole conversation instead.+ """+ texts: list[str] = []+ for message in (output or {}).get("messages") or []:+ if not isinstance(message, dict) or message.get("role") != "assistant":+ continue+ content = message.get("content")+ if isinstance(content, str):+ texts.append(content)+ elif isinstance(content, list):+ texts.extend(+ block["text"]+ for block in content+ if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str)+ )+ if not texts:+ last_message = (output or {}).get("last_message")+ if isinstance(last_message, str):+ texts.append(last_message)+ return "\n\n".join(text for text in texts if text)+++def tool_output_text(output: str) -> str:+ """The text a tool returned, without the MCP content blocks that the log keeps around it."""+ try:+ payload = json.loads(output)+ except (TypeError, ValueError):+ return output+ if isinstance(payload, dict):+ payload = payload.get("content")+ if not isinstance(payload, list):+ return output+ texts = [+ block["text"]+ for block in payload+ if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str)+ ]+ return "\n".join(texts) if texts else output+++def _query_node_kind(node: object) -> str | None:+ """``sql`` for a ``HogQLQuery`` at any wrapper depth, ``insight`` for any other query kind."""+ for _ in range(3):+ if not isinstance(node, dict):+ return None+ kind = node.get("kind")+ if kind == "HogQLQuery":+ return "sql"+ if not isinstance(node.get("source"), dict):+ return "insight" if isinstance(kind, str) and kind else None+ node = node["source"]+ return None+++def _collect_saved_insight_kinds(node: object, kinds: dict[str, str]) -> None:+ if isinstance(node, dict):+ short_id = node.get("short_id")+ kind = _query_node_kind(node.get("query"))+ if isinstance(short_id, str) and short_id and kind is not None:+ kinds[short_id] = kind+ for value in node.values():+ _collect_saved_insight_kinds(value, kinds)+ elif isinstance(node, list):+ for value in node:+ _collect_saved_insight_kinds(value, kinds)+++# Both match JSON and the TOON text that exec returns in its default output format.+_SHORT_ID_RE = re.compile(r"""(?:\bshort_id["']?\s*:\s*["']?|/insights/)(\w+)""")+_QUERY_KIND_RE = re.compile(r"""\bkind["']?\s*:\s*["']?(\w+)""")+++def saved_insight_kinds(calls: list[ToolCall]) -> dict[str, str]:+ """The query kind (``insight`` or ``sql``) of each saved insight whose query a tool call showed.++ A saved insight can hold a ``HogQLQuery``, so its link alone does not tell a typed chart from a+ SQL chart. JSON output is read exactly. Text output is read only when it names one insight.+ """+ kinds: dict[str, str] = {}+ for call in calls:+ if call.is_error:+ continue+ text = tool_output_text(call.output)+ try:+ _collect_saved_insight_kinds(json.loads(text), kinds)+ except (TypeError, ValueError):+ pass+ short_ids = set(_SHORT_ID_RE.findall(text)) - {"new"}+ if len(short_ids) != 1 or next(iter(short_ids)) in kinds:+ continue+ query = call.input.get("query")+ query_kinds = set(_QUERY_KIND_RE.findall(text + (json.dumps(query) if isinstance(query, dict) else "")))+ if "HogQLQuery" in query_kinds:+ kinds[next(iter(short_ids))] = "sql"+ elif any(kind.endswith("Query") for kind in query_kinds):+ kinds[next(iter(short_ids))] = "insight"+ return kinds+++def _chart_tag_kind(paragraph: str, insight_kinds: Mapping[str, str]) -> str | None:+ match = _CHART_TAG_RE.match(paragraph)+ if match is None:+ return None+ tag, attrs, body = match.group(1).lower(), dict(_TAG_ATTR_RE.findall(match.group(2))), match.group(3).strip()+ if attrs.get("display") != "block":+ return None+ if tag == "insight":+ short_id = attrs.get("id")+ return insight_kinds.get(short_id, "saved") if short_id else None+ return "sql" if body else None+++def chart_references(+ text: str, project_ids: frozenset[str] = frozenset(), insight_kinds: Mapping[str, str] | None = None+) -> list[str]:+ """Kinds of the full-size charts in ``text``, in order.++ ``insight`` is a typed query, ``sql`` is a SQL query, and ``saved`` is a saved insight that is+ not in ``insight_kinds``, so its query kind is unknown. Only a paragraph that holds nothing but a+ chart link or a block tag counts, and code blocks are skipped, because Desktop draws nothing else+ as a chart card. When ``project_ids`` is not empty, a link to any other project is skipped.+ """+ insight_kinds = insight_kinds or {}+ kinds: list[str] = []+ for paragraph in re.split(r"\n\s*\n", _CODE_FENCE_RE.sub("", text)):+ paragraph = paragraph.strip()+ match = _CHART_LINK_RE.match(paragraph)+ if match is None:+ tag_kind = _chart_tag_kind(paragraph, insight_kinds)+ if tag_kind is not None:+ kinds.append(tag_kind)+ continue+ link_kind = _chart_link_kind(match.group(1), project_ids, insight_kinds)+ if link_kind is not None:+ kinds.append(link_kind)+ return kinds+++def _hogql_from_query_node(raw: str) -> str | None:+ try:+ node = json.loads(raw)+ except ValueError:+ return None+ for _ in range(3):+ if not isinstance(node, dict):+ return None+ if node.get("kind") == "HogQLQuery" and isinstance(node.get("query"), str):+ return node["query"]+ node = node.get("source")+ return None+++def _chart_link_kind(url: str, project_ids: frozenset[str], insight_kinds: Mapping[str, str]) -> str | None:+ """Mirror ``parsePostHogObjectUrl`` in ``products/desktop/packages/core/src/posthog-objects/objectUrls.ts``.++ A saved insight (``/insights/<id>``, ``/i/<id>``) takes its kind from ``insight_kinds``. ``/sql?open_query=`` and an+ ``/insights/new#q=`` link that holds a ``HogQLQuery`` are SQL charts; an unsaved typed query is no+ chart. Desktop also requires the app host, which the eval cannot know because the agent's links+ and the tool results use different hosts, so only the project is checked.+ """+ parts = urlsplit(url)+ segments = [segment for segment in parts.path.split("/") if segment]+ if segments[:1] == ["project"]:+ if len(segments) < 2 or (project_ids and segments[1] not in project_ids):+ return None+ segments = segments[2:]+ if segments == ["sql"]:+ query = parse_qs(parts.query).get("open_query", [""])[0].strip()+ if query.startswith("{"):+ query = (_hogql_from_query_node(query) or "").strip()+ return "sql" if query else None+ if segments == ["insights", "new"]:+ node = parse_qs(parts.fragment).get("q", [""])[0]+ return "sql" if (_hogql_from_query_node(node) or "").strip() else None+ if len(segments) >= 2 and segments[0] in ("insights", "i") and segments[1] != "new":+ return insight_kinds.get(segments[1], "saved")+ return None+++# Inputs the exec wrapper accepts that are not part of the query, so a render that leaves them out still matches.+_NON_QUERY_INPUT_KEYS = frozenset({"output_format"})+++def _query_input(call: ToolCall) -> dict:+ return {key: value for key, value in call.input.items() if key not in _NON_QUERY_INPUT_KEYS}+++def _render_matches_query(render: ToolCall, calls: list[ToolCall]) -> bool:+ """True when a ``render-ui`` call passes the input of a successful call to the same query tool.++ ``render-ui`` succeeds for any input because it only emits a render directive, and the chart then+ fetches its own data. An empty or invented input therefore shows an error, not a chart.+ """+ tool_name = _rendered_tool_name(render)+ tool_input = render.input.get("tool_input")+ if not isinstance(tool_input, dict) or not tool_input:+ return False+ rendered = {key: value for key, value in tool_input.items() if key not in _NON_QUERY_INPUT_KEYS}+ return any(+ not call.is_error and call.name == tool_name and _query_input(call) == rendered+ for call in calls+ if call.position < render.position+ )+++def _chart_paths(output: dict | None, parser: LogParser, tools: set[str]) -> list[str]:+ """How the user saw a chart: ``render-ui``, ``inline`` (the client's UI app), or a ``chart_references`` kind."""+ paths: list[str] = []+ calls = parser.get_tool_calls()+ for call in _render_ui_calls(parser):+ if not call.is_error and _rendered_tool_name(call) in tools and _render_matches_query(call, calls):+ paths.append("render-ui")+ for call in calls:+ if not call.is_error and call.name in tools and INLINE_VIEW_NOTE in call.output:+ paths.append("inline")+ project_ids = frozenset(match for call in calls for match in _PROJECT_URL_RE.findall(call.output))+ paths.extend(chart_references(assistant_text(output), project_ids, saved_insight_kinds(calls)))+ return paths+++def _chart_tools(expected: dict | None, scorer_name: str) -> set[str] | None:+ spec = _read_spec(expected, scorer_name)+ if spec is None:+ return None+ allowed = spec.get("tool_name_any_of")+ return {t for t in allowed if isinstance(t, str) and t} if isinstance(allowed, list) else set()+++class ChartShown(Scorer):+ """Binary: did the user get the analytics result as a chart?++ ``expected = {"chart_shown": {"tool_name_any_of": ["query-trends"]}}``++ Any chart path counts: ``render-ui`` with a listed query tool, a listed query tool result+ that the client already displays inline (``INLINE_VIEW_NOTE``), or a full-size chart link or+ block tag in the answer (a saved insight or a SQL query). ``ChartFromTypedQuery``+ grades which of these the agent chose. ``None`` if the spec is absent.+ """++ def _name(self) -> str:+ return "chart_shown"++ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:+ tools = _chart_tools(expected, self._name())+ if tools is None:+ return Score(name=self._name(), score=None, metadata={"reason": f"No {self._name()} spec on case"})+ parser = _build_parser(output)+ if parser is None:+ return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})+ paths = _chart_paths(output, parser, tools)+ if paths:+ return Score(name=self._name(), score=1.0, metadata={"paths": paths})+ return Score(+ name=self._name(),+ score=0.0,+ metadata={"reason": "No chart was shown for the expected query tools", "expected_any_of": sorted(tools)},+ )+++class ChartFromTypedQuery(Scorer):+ """Binary: when a chart was shown, did it come from the typed query rather than rewritten SQL?++ ``expected = {"chart_from_typed_query": {"tool_name_any_of": ["query-trends"]}}``++ An agent that answers with ``query-trends`` and then links a hand-written SQL chart shows+ numbers computed by a different query, without PostHog's standard definitions such as the+ project timezone or test-account filtering. Score 1.0 when any chart comes from+ ``render-ui``, the inline view, or a saved insight that holds a typed query; 0.0 when every+ chart is a SQL query. ``None`` when no chart was shown (graded by ``ChartShown``), when the only+ other charts are saved insights whose query kind no tool call showed, or the spec is absent.+ """++ def _name(self) -> str:+ return "chart_from_typed_query"++ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:+ tools = _chart_tools(expected, self._name())+ if tools is None:+ return Score(name=self._name(), score=None, metadata={"reason": f"No {self._name()} spec on case"})+ parser = _build_parser(output)+ if parser is None:+ return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})+ paths = _chart_paths(output, parser, tools)+ if not paths:+ return Score(name=self._name(), score=None, metadata={"reason": "No chart was shown"})+ if any(path not in ("sql", "saved") for path in paths):+ return Score(name=self._name(), score=1.0, metadata={"paths": paths})+ if "saved" in paths:+ return Score(+ name=self._name(),+ score=None,+ metadata={"paths": paths, "reason": "No tool call showed the query of the linked saved insight"},+ )+ return Score(name=self._name(), score=0.0, metadata={"paths": paths})+++_TABLE_SEPARATOR_RE = re.compile(r"^(?=.*\|)\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$")+++def markdown_table_row_counts(text: str) -> list[int]:+ """Count the data rows of each Markdown table in ``text``, in order."""+ lines = text.splitlines()+ counts: list[int] = []+ index = 1+ while index < len(lines):+ if "|" in lines[index - 1] and _TABLE_SEPARATOR_RE.match(lines[index]):+ rows = 0+ index += 1+ while index < len(lines) and "|" in lines[index] and lines[index].strip():+ rows += 1+ index += 1+ counts.append(rows)+ index += 1+ return counts+++class NoMarkdownDataTable(Scorer):+ """Binary: did the agent avoid restating the query result as a Markdown table?++ ``expected = {"no_markdown_data_table": {}}`` or ``{"no_markdown_data_table": {"max_rows": 3}}``++ A chart already shows the series, so a table that retypes it adds length and hides+ the conclusion. Small summary tables (at most ``max_rows`` data rows, default 3, such as+ this period against the previous one) still pass. Score 0.0 when any table in the agent's+ messages is longer. ``None`` if the spec is absent or the agent wrote no text.+ """++ DEFAULT_MAX_ROWS = 3++ def _name(self) -> str:+ return "no_markdown_data_table"++ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:+ spec = _read_spec(expected, self._name())+ if spec is None:+ return Score(name=self._name(), score=None, metadata={"reason": f"No {self._name()} spec on case"})++ text = assistant_text(output)+ if not text.strip():+ return Score(name=self._name(), score=None, metadata={"reason": "No agent text"})++ max_rows = spec.get("max_rows", self.DEFAULT_MAX_ROWS)+ row_counts = markdown_table_row_counts(text)+ too_long = [rows for rows in row_counts if rows > max_rows]+ return Score(+ name=self._name(),+ score=0.0 if too_long else 1.0,+ metadata={"table_row_counts": row_counts, "max_rows": max_rows},+ )+++class _TableSources(TraversingVisitor):+ def __init__(self) -> None:+ self.tables: list[str] = []+ self.ctes: set[str] = set()++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ self.ctes.update(name.lower() for name in node.ctes or {})+ super().visit_select_query(node)++ def visit_join_expr(self, node: ast.JoinExpr) -> None:+ if isinstance(node.table, ast.Field):+ self.tables.append(".".join(str(part) for part in node.table.chain).lower())+ super().visit_join_expr(node)+++def is_catalog_sql(call: ToolCall) -> bool:+ """True when an ``execute-sql`` call reads only PostHog entity tables (``system.*``).++ Catalog lookups, such as checking for a saved insight or an action, are justified SQL+ under the routing rules, so they never count as an analytics query. A query that the+ HogQL parser cannot read counts as analytics SQL.+ """+ query = call.input.get("query")+ if not isinstance(query, str):+ return False+ try:+ sources = _TableSources()+ sources.visit(parse_select(query))+ except BaseHogQLError:+ return False+ tables = [table for table in sources.tables if table not in sources.ctes]+ return bool(tables) and all(table.startswith("system.") for table in tables)diff --git a/products/posthog_ai/evals/product_analytics/eval_chart_not_table.py b/products/posthog_ai/evals/product_analytics/eval_chart_not_table.pynew file mode 100644index 000000000000..de7efdda29ec--- /dev/null+++ b/products/posthog_ai/evals/product_analytics/eval_chart_not_table.py@@ -0,0 +1,106 @@+"""Does the agent answer a chart-shaped analytics question with a chart, not a Markdown table?++Agents often run the right query and then retype every data point as a Markdown table, or+answer with ``execute-sql``, which has no chart at all. Each case asks a question whose+natural answer is a time series, a breakdown, a funnel, or a retention grid. The agent passes+when the user sees the result as a chart and the final message states the conclusion+without a long table.++``ChartShown`` accepts every chart path a harness offers: the MCP client shows the query+result inline, the agent calls ``render-ui`` with the same query tool and input, or the answer+holds a full-size chart link or block tag (the PostHog Desktop syntax). ``ChartFromTypedQuery``+then fails a chart that rewrites the typed result as SQL. PostHog Desktop draws a chart card+only for a saved insight or a HogQL query, so an agent that does not save an insight can+reach a chart only through SQL; this scorer measures how often that happens.++The ``table_request_keeps_typed_runner`` control asks for a table explicitly. A table is+then a correct answer, but the request alone does not justify SQL.++Run on the latest Claude Opus and GPT Sol models, one runtime per run:++ hogli evals eval_chart_not_table --agent-runtime claude --agent-model claude-opus-5-5 --trials 3+ hogli evals eval_chart_not_table --agent-runtime codex --agent-model gpt-6.1-sol --trials 3+"""++from __future__ import annotations++from products.posthog_ai.eval_harness.base import SandboxedPublicEval+from products.posthog_ai.eval_harness.config import SandboxedEvalCase+from products.posthog_ai.eval_harness.harness.context import EvalContext+from products.posthog_ai.eval_harness.scorers import NoToolCall+from products.posthog_ai.evals.cli_mcp.scorers import (+ ChartFromTypedQuery,+ ChartShown,+ DidNotCiteRawSqlAfterTypedQuery,+ NoMarkdownDataTable,+)+from products.posthog_ai.evals.product_analytics.scorers import INSIGHT_WRITE_TOOLS, AnswerFromTypedQuery+++def _chart_case(name: str, prompt: str, tool: str) -> SandboxedEvalCase:+ return SandboxedEvalCase(+ name=name,+ prompt=prompt,+ expected={+ "answer_from_typed_query": {},+ "chart_shown": {"tool_name_any_of": [tool]},+ "chart_from_typed_query": {"tool_name_any_of": [tool]},+ "no_markdown_data_table": {},+ "did_not_cite_raw_sql_after_typed_query": {"tool": tool},+ },+ )+++async def eval_chart_not_table(ctx: EvalContext) -> None:+ cases = [+ _chart_case(+ "weekly_signups_trend",+ "How have weekly signups changed over the last 12 weeks?",+ "query-trends",+ ),+ _chart_case(+ "daily_uploads_by_file_type",+ "Show daily uploaded_file counts by file_type for the last 30 days.",+ "query-trends",+ ),+ _chart_case(+ "weekly_file_activity_scorecard",+ "Give me weekly uploads, downloads, and signups for the last 8 weeks.",+ "query-trends",+ ),+ _chart_case(+ "pageviews_by_country",+ "Which countries sent the most pageviews in the last 30 days?",+ "query-trends",+ ),+ _chart_case(+ "pricing_to_upgrade_funnel",+ "What does our funnel from a /pricing/ pageview to signed_up to upgraded_plan look like over the last "+ "30 days?",+ "query-funnel",+ ),+ _chart_case(+ "weekly_signup_retention",+ "How well do users who signed up in the last 8 weeks keep coming back each week?",+ "query-retention",+ ),+ SandboxedEvalCase(+ name="table_request_keeps_typed_runner",+ prompt="List the weekly signup counts for the last 8 weeks as a table.",+ expected={"answer_from_typed_query": {}},+ ),+ ]++ await SandboxedPublicEval(+ experiment_name="sandboxed-chart-not-table-cli",+ cases=cases,+ scorers=[+ NoToolCall(forbidden=INSIGHT_WRITE_TOOLS, name="no_persistent_insight_save"),+ AnswerFromTypedQuery(),+ ChartShown(),+ ChartFromTypedQuery(),+ NoMarkdownDataTable(),+ DidNotCiteRawSqlAfterTypedQuery(),+ ],+ ctx=ctx,+ )diff --git a/products/posthog_ai/evals/product_analytics/eval_typed_runner_misses.py b/products/posthog_ai/evals/product_analytics/eval_typed_runner_misses.pynew file mode 100644index 000000000000..8b52c9c9580e--- /dev/null+++ b/products/posthog_ai/evals/product_analytics/eval_typed_runner_misses.py@@ -0,0 +1,206 @@+"""Product-analytics requests that agents answer with ``execute-sql`` instead of a typed runner.++Each case reproduces, on Hedgebox, a request shape that the online evaluation+"MCP SQL: typed query runner choice" fails most often on production MCP traces:++- Several metrics in one question ("counts, uniques and a filtered count"), which agents pack+ into one SQL query with ``countIf`` and ``uniqIf`` instead of trends series.+- A count of events that carry a property, a single-day unique-user count, and active users+ without internal accounts, which agents write as SQL with hand-built filters.+- Breakdowns, period comparisons, percentiles, and ratios, which trends expresses with a+ breakdown, ``compareFilter``, property math, and a formula.+- Funnels and return rates computed with ``uniqIf`` per event, which drop the step order and+ conversion window that ``query-funnel`` and ``query-retention`` keep.+- A justified ``system.*`` lookup, followed by an analysis that stays on SQL.++``AnswerFromTypedQuery`` checks which data the answer reports: its values must come from a typed+query result. A SQL query that checks a typed result is allowed, which is looser than the online+judge, because the judge fails any SQL call against event data.+``eval_justified_sql_controls`` holds requests where SQL is the right tool, so a guidance+change that removes these misses cannot pass by sending everything to typed runners.++Run the suites on the latest Claude Opus and GPT Sol models, one runtime per run, and compare+the two runs paired by case:++ hogli evals eval_typed_runner_misses --agent-runtime claude --agent-model claude-opus-5-5 --trials 3+ hogli evals eval_typed_runner_misses --agent-runtime codex --agent-model gpt-6.1-sol --trials 3+"""++from __future__ import annotations++from typing import Unpack++from products.posthog_ai.eval_harness.base import SandboxedPublicEval+from products.posthog_ai.eval_harness.config import SandboxedEvalCase+from products.posthog_ai.eval_harness.harness.context import EvalContext+from products.posthog_ai.eval_harness.scorers import AnswerToolCallNot, NoToolCall+from products.posthog_ai.evals.cli_mcp.scorers import FirstRelevantTool+from products.posthog_ai.evals.product_analytics.scorers import (+ INSIGHT_WRITE_TOOLS,+ TYPED_QUERY_TOOLS,+ AnswerFromTypedQuery,+ InsightShape,+ InsightShapeSpec,+)++from .eval_sql_shaped_routing import ANALYSIS_QUERY_TOOLS+++def _case(+ name: str, prompt: str, *, first_tool: str | None = None, **shape: Unpack[InsightShapeSpec]+) -> SandboxedEvalCase:+ expected: dict[str, object] = {}+ if first_tool is None:+ expected["answer_from_typed_query"] = {}+ else:+ expected["first_relevant_tool"] = {"tool": first_tool}+ if shape:+ expected["insight_shape"] = shape+ return SandboxedEvalCase(name=name, prompt=prompt, expected=expected)+++async def eval_typed_runner_misses(ctx: EvalContext) -> None:+ cases = [+ _case(+ "scorecard_counts_and_uploaders",+ "For the last 7 days, give me the number of uploaded_file events, the number of downloaded_file events, "+ "and how many unique users uploaded a file.",+ tool="query-trends",+ events=["uploaded_file", "downloaded_file", "uploaded_file"],+ math=["total", "dau"],+ ),+ _case(+ "count_with_file_name_present",+ "How many uploaded_file events were there in the last 30 days, and how many of them had a file_name?",+ tool="query-trends",+ events=["uploaded_file", "uploaded_file"],+ filtered_on=["file_name"],+ ),+ _case(+ "unique_signups_yesterday",+ "How many unique users signed up yesterday?",+ tool="query-trends",+ events=["signed_up"],+ math=["dau"],+ ),+ _case(+ "active_users_excluding_internal",+ "What were our weekly and monthly active users over the last 30 days, excluding internal and test "+ "accounts? Count anyone who fired logged_in.",+ tool="query-trends",+ math=["weekly_active", "monthly_active"],+ filter_test_accounts=True,+ ),+ _case(+ "downloads_by_file_type_counts_and_uniques",+ "Break down yesterday's downloaded_file events by file_type, with the event count and the number of "+ "unique users for each type.",+ tool="query-trends",+ breakdown="file_type",+ math=["total", "dau"],+ ),+ _case(+ "uploads_by_os_vs_previous_week",+ "Compare uploaded_file events per operating system over the last 7 days with the 7 days before.",+ tool="query-trends",+ breakdown="$os",+ compare=True,+ events=["uploaded_file"],+ ),+ _case(+ "net_plan_upgrades_formula",+ "What's the net number of plan upgrades minus downgrades per week over the last 8 weeks?",+ tool="query-trends",+ events=["upgraded_plan", "downgraded_plan"],+ formula=True,+ ),+ _case(+ "file_size_percentiles_vs_previous_period",+ "What are the median and 95th percentile file_size_b of uploaded files by file_type over the last 7 "+ "days, compared with the 7 days before?",+ tool="query-trends",+ breakdown="file_type",+ compare=True,+ math=["median", "p95"],+ ),+ _case(+ "pricing_session_share_weekly",+ "What share of weekly sessions with a pageview included a pageview of /pricing/, over the last 8 weeks?",+ tool="query-trends",+ events=["$pageview", "$pageview"],+ math=["unique_session"],+ formula=True,+ filtered_on=["$pathname", "$current_url"],+ ),+ _case(+ "dashboard_tile_signup_funnel",+ "I'm preparing an operations dashboard tile. Compute the 30-day signup funnel: a pageview of /signup/, "+ "then signed_up, then uploaded_file, then upgraded_plan. Don't save anything yet.",+ tool="query-funnel",+ event_sequence=["$pageview", "signed_up", "uploaded_file", "upgraded_plan"],+ ),+ _case(+ "signup_to_upload_last_14_days_vs_prior",+ "Did conversion from signed_up to uploaded_file change in the last 14 days compared with the 14 days "+ "before?",+ tool="query-funnel",+ event_sequence=["signed_up", "uploaded_file"],+ ),+ _case(+ "day1_day7_return_rates",+ "For users who first fired logged_in in the last 30 days, what are their day-1 and day-7 return rates?",+ tool="query-retention",+ ),+ # The system.* lookup is justified SQL and does not count as analytics SQL.+ _case(+ "existing_insight_check_then_query",+ "Check whether we already have a saved insight for weekly logins by browser. If we don't, show me "+ "weekly logged_in counts by $browser over the last 8 weeks.",+ tool="query-trends",+ breakdown="$browser",+ events=["logged_in"],+ ),+ ]++ await SandboxedPublicEval(+ experiment_name="sandboxed-typed-runner-misses-cli",+ cases=cases,+ scorers=[+ NoToolCall(forbidden=INSIGHT_WRITE_TOOLS, name="no_persistent_insight_save"),+ AnswerFromTypedQuery(),+ AnswerToolCallNot("execute-sql", TYPED_QUERY_TOOLS),+ InsightShape(),+ ],+ ctx=ctx,+ )+++async def eval_justified_sql_controls(ctx: EvalContext) -> None:+ cases = [+ _case(+ "sql_explicit_hogql_request",+ "Write and run a HogQL query that counts uploaded_file events per day over the last 14 days.",+ first_tool="execute-sql",+ ),+ _case(+ "sql_latest_event_timestamp",+ "When did the most recent uploaded_file event arrive? Give me its exact timestamp.",+ first_tool="execute-sql",+ ),+ _case(+ "sql_gap_between_first_and_second_upload",+ "For each user who uploaded a file in the last 30 days, compute the hours between their first and "+ "second uploaded_file event, and show the 10 users with the longest gap.",+ first_tool="execute-sql",+ ),+ ]++ await SandboxedPublicEval(+ experiment_name="sandboxed-justified-sql-controls-cli",+ cases=cases,+ scorers=[+ NoToolCall(forbidden=INSIGHT_WRITE_TOOLS, name="no_persistent_insight_save"),+ FirstRelevantTool(relevant_tools=ANALYSIS_QUERY_TOOLS),+ ],+ ctx=ctx,+ )diff --git a/products/posthog_ai/evals/product_analytics/scorers.py b/products/posthog_ai/evals/product_analytics/scorers.pyindex e19b3615f64c..2f70f67d1460 100644--- a/products/posthog_ai/evals/product_analytics/scorers.py+++ b/products/posthog_ai/evals/product_analytics/scorers.py@@ -13,8 +13,11 @@ and only then runs the actual query tool.from __future__ import annotations+import re+import jsonfrom collections.abc import Iterable-from typing import Any+from decimal import Decimal, InvalidOperation+from typing import Any, TypedDictfrom pydantic import BaseModel@@ -29,6 +32,7 @@ from products.posthog_ai.eval_harness.scorers import (JudgedScorer,)from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer+from products.posthog_ai.evals.cli_mcp.scorers import assistant_text, is_catalog_sql, tool_output_textQUERY_TRENDS_TOOL_NAME = "query-trends"QUERY_RETENTION_TOOL_NAME = "query-retention"@@ -732,6 +736,22 @@ _ANSWER_TOOL_SCHEMAS: dict[str, type[BaseModel]] = {}+class InsightShapeSpec(TypedDict, total=False):+ """The keys ``InsightShape`` reads from ``expected["insight_shape"]``."""++ tool: str | list[str]+ display: str | list[str]+ compare: bool+ funnel_viz: str | list[str]+ breakdown: str | None+ events: list[str | int]+ event_sequence: list[str | int]+ math: list[str]+ formula: bool+ filter_test_accounts: bool+ filtered_on: list[str]++class InsightShape(Scorer):"""Binary: did the agent's answer query have the expected skeleton?@@ -751,9 +771,14 @@ class InsightShape(Scorer):- ``breakdown``: the breakdown property name, or ``None`` to require no breakdown.- ``events``: the multiset of series events, in any order.- ``event_sequence``: the series events in order (funnel steps).+ - ``math``: math types that must each appear on at least one series; an omitted+ math counts as ``total``.+ - ``formula``: whether a trends query combines its series with a formula.+ - ``filter_test_accounts``: whether the query excludes internal and test accounts.+ - ``filtered_on``: property keys of which at least one must appear in a query or series filter.The keys after ``tool`` describe a series query, so they are checked only when- the chosen tool is ``query-trends`` or ``query-funnel``. Filters, math, and date+ the chosen tool is ``query-trends`` or ``query-funnel``. Property filters and dateranges are left to the schema-alignment judges."""@@ -793,6 +818,10 @@ class InsightShape(Scorer):"funnel_viz": _actual_funnel_viz(query, tool_name),"breakdown": _actual_breakdown(query),"event_sequence": _actual_events(query),+ "math": _actual_math(query),+ "formula": _actual_formula(query, tool_name),+ "filter_test_accounts": query.get("filterTestAccounts") is True,+ "filtered_on": sorted(_filtered_properties(query)),}actual["events"] = sorted(actual["event_sequence"], key=str)checked = shape if tool_name in SERIES_QUERY_TOOLS else {"tool": shape.get("tool")}@@ -835,8 +864,12 @@ def _shape_field_matches(key: str, expected_value: Any, actual: dict[str, Any])if key in ("tool", "display", "funnel_viz"):accepted = expected_value if isinstance(expected_value, list | tuple | set) else [expected_value]return actual[key] in {str(item) for item in accepted}- if key in ("compare", "breakdown"):+ if key in ("compare", "breakdown", "formula", "filter_test_accounts"):return actual[key] == expected_value+ if key == "filtered_on":+ return isinstance(expected_value, list) and bool(set(expected_value) & set(actual["filtered_on"]))+ if key == "math":+ return isinstance(expected_value, list) and set(expected_value) <= set(actual["math"])if key == "events":return isinstance(expected_value, list) and sorted(expected_value, key=str) == actual["events"]if key == "event_sequence":@@ -882,17 +915,159 @@ def _actual_breakdown(query: dict[str, Any]) -> str | None:return str(breakdown) if breakdown else None-def _actual_events(query: dict[str, Any]) -> list[str | float | None]:- events: list[str | float | None] = []+def _series_nodes(query: dict[str, Any]) -> list[dict[str, Any]]:+ nodes: list[dict[str, Any]] = []for node in query.get("series") or []:if not isinstance(node, dict):continueinner = node.get("nodes") if node.get("kind") == "GroupNode" else [node]- for entry in inner or []:- if not isinstance(entry, dict):- continue- if "event" in entry:- events.append(entry.get("event"))- elif entry.get("kind") == "ActionsNode":- events.append(entry.get("id"))+ nodes.extend(entry for entry in inner or [] if isinstance(entry, dict))+ return nodes+++def _actual_math(query: dict[str, Any]) -> list[str]:+ # A GroupNode's own math applies to the whole group; the engine ignores the math of its inner nodes.+ series = [node for node in query.get("series") or [] if isinstance(node, dict)]+ return sorted({str(node.get("math") or "total") for node in series})+++def _filtered_properties(query: dict[str, Any]) -> set[str]:+ """Property keys used in the query's own filters or in any series filter."""+ keys: set[str] = set()++ def collect(filters: Any) -> None:+ if isinstance(filters, dict):+ if isinstance(filters.get("key"), str):+ keys.add(filters["key"])+ collect(filters.get("values"))+ elif isinstance(filters, list):+ for item in filters:+ collect(item)++ collect(query.get("properties"))+ for node in query.get("series") or []:+ if isinstance(node, dict):+ collect(node.get("properties"))+ for node in _series_nodes(query):+ collect(node.get("properties"))+ return keys+++def _actual_formula(query: dict[str, Any], tool_name: str) -> bool | None:+ if tool_name != QUERY_TRENDS_TOOL_NAME:+ return None+ trends_filter = query.get("trendsFilter")+ if not isinstance(trends_filter, dict):+ return False+ # The assistant schema uses ``formulaNodes``; saved TrendsFilter queries can carry the older keys.+ return any(trends_filter.get(key) for key in ("formulaNodes", "formulas", "formula"))+++def _actual_events(query: dict[str, Any]) -> list[str | float | None]:+ events: list[str | float | None] = []+ for entry in _series_nodes(query):+ if "event" in entry:+ events.append(entry.get("event"))+ elif entry.get("kind") == "ActionsNode":+ events.append(entry.get("id"))return events+++# Text that holds digits but no reported values: links (URL encoding such as ``%20``), ISO dates and+# times, clock times, and "Oct 12" style dates.+_NON_VALUE_RES = (+ re.compile(r"https?://\S+"),+ re.compile(r"\d{4}-\d{2}-\d{2}(?:[T ][\d:.]+)?(?:Z|[+-]\d{2}:?\d{2})?"),+ re.compile(r"\b\d{1,2}:\d{2}(?::\d{2})?\b"),+ re.compile(r"\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}\b", re.IGNORECASE),+ re.compile(r"\b\d{1,2}\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\b", re.IGNORECASE),+)+# A minus sign counts only at the start of a value, so the range "6-16" reads as 6 and 16, not 6 and -16.+_VALUE_RE = re.compile(r"(?<![\w.])[-\u2212]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?(?![\w])")+++def reported_values(text: str, ignore: frozenset[str] = frozenset(), *, skip_years: bool = True) -> set[str]:+ """Data values in ``text``, normalized so ``1,234``, ``1234`` and ``1234.0`` compare equal.++ Single-digit integers and years written without a thousands separator are dropped, because they+ also appear as window sizes, step numbers, and dates, so they cannot show where a value came from.+ An answer whose only values are of this kind gets no score rather than a wrong one. Result tables+ pass ``skip_years=False``, because a year-like value there is a data cell.+ """+ for pattern in _NON_VALUE_RES:+ text = pattern.sub(" ", text)+ values: set[str] = set()+ for raw in _VALUE_RE.findall(text):+ try:+ number = Decimal(raw.replace(",", "").replace("\u2212", "-"))+ except InvalidOperation:+ continue+ if number == number.to_integral_value() and (+ abs(number) < 10 or (skip_years and "," not in raw and 1900 <= number <= 2100)+ ):+ continue+ value = format(number.normalize(), "f")+ if value not in ignore:+ values.add(value)+ return values+++def _result_text(call: ToolCall) -> str:+ """The result rows of a query tool call, without the query, notes, and examples the tool adds around them."""+ text = tool_output_text(call.output)+ try:+ payload = json.loads(text)+ except (TypeError, ValueError):+ payload = None+ if isinstance(payload, dict) and "results" in payload:+ return json.dumps(payload["results"])+ rows = [line for line in text.splitlines() if "|" in line]+ return "\n".join(rows) if rows else text+++class AnswerFromTypedQuery(Scorer):+ """Binary: does the answer report data from a typed query result rather than from SQL?++ ``expected = {"answer_from_typed_query": {}}``++ Compares the values in the agent's messages with the values each tool returned. The case+ passes when the answer reports at least one value from a successful typed query result. A+ SQL query that checks a typed result is allowed, and the answer can mention its values too.+ The case fails when the reported values come only from analytics SQL results, or when+ analytics SQL ran and no typed query did. ``system.*`` catalog lookups are not analytics SQL.+ ``None`` when the answer has no value traceable to either source, or the spec is absent.+ """++ def _name(self) -> str:+ return "answer_from_typed_query"++ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:+ if not isinstance(expected, dict) or not isinstance(expected.get(self._name()), dict):+ return Score(name=self._name(), score=None, metadata={"reason": f"No {self._name()} key on case"})+ parser = parser_for(output)+ if parser is None:+ return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})++ calls = [call for call in parser.get_tool_calls() if not call.is_error]+ typed_calls = [call for call in calls if call.name in TYPED_QUERY_TOOLS]+ sql_calls = [call for call in calls if call.name == "execute-sql" and not is_catalog_sql(call)]+ if not typed_calls:+ if sql_calls:+ return Score(name=self._name(), score=0.0, metadata={"reason": "Only analytics SQL ran"})+ return Score(name=self._name(), score=None, metadata={"reason": "No query ran"})++ ignore = frozenset(reported_values(user_prompt(output)))+ answer = reported_values(assistant_text(output), ignore)+ typed = set().union(*(reported_values(_result_text(call), skip_years=False) for call in typed_calls))+ sql = set().union(*(reported_values(_result_text(call), skip_years=False) for call in sql_calls))+ sql_only = sql - typed+ typed_hits = sorted(answer & typed)+ sql_hits = sorted(answer & sql_only)+ metadata = {"typed_values_reported": typed_hits, "sql_only_values_reported": sql_hits}+ if typed_hits:+ return Score(name=self._name(), score=1.0, metadata=metadata)+ if sql_hits:+ return Score(name=self._name(), score=0.0, metadata=metadata)+ return Score(+ name=self._name(), score=None, metadata={**metadata, "reason": "No reported value matches a query result"}+ )