:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(dashboards): add an agent eval for dashboard widget tools (#120220)
Thomas Obermüllerce345fca75be
diff --git a/.agents/skills/manage-dashboard-widgets/references/mcp.md b/.agents/skills/manage-dashboard-widgets/references/mcp.mdindex 0cd08bbf92b1..6265633a4c1a 100644--- a/.agents/skills/manage-dashboard-widgets/references/mcp.md+++ b/.agents/skills/manage-dashboard-widgets/references/mcp.md@@ -55,3 +55,4 @@ UI multi-select add uses `POST .../widgets/batch/` (not PATCH dashboard `tiles[]1. Regenerate OpenAPI (`hogli build:openapi`) — updates MCP tool schemas that embed supported type lists2. Run `hogli test services/mcp/tests/tools/dashboards.integration.test.ts`3. Update unit schema snapshots if batch-add help text lists types: `hogli test services/mcp/tests/unit/tool-schema-snapshots.test.ts`+4. To check that agents can configure the new type, add a case to `products/dashboards/evals/eval_dashboard_widgets.py` and run `hogli evals eval_dashboard_widgets` (see the `writing-evals` skill)diff --git a/products/dashboards/evals/eval_dashboard_widgets.py b/products/dashboards/evals/eval_dashboard_widgets.pynew file mode 100644index 000000000000..45cf2a69aa55--- /dev/null+++ b/products/dashboards/evals/eval_dashboard_widgets.py@@ -0,0 +1,118 @@+from __future__ import annotations++from typing import Any++from products.dashboards.backend.models.dashboard import Dashboard+from products.dashboards.backend.models.dashboard_tile import DashboardTile+from products.dashboards.backend.models.dashboard_widget import DashboardWidget+from products.dashboards.backend.widget_specs.registry import validate_widget_config+from products.dashboards.evals.offline_config import offline_suite+from products.dashboards.evals.scorers import SavedWidgetState, WidgetFirstWriteValid+from products.posthog_ai.eval_harness.base import SandboxedPrivateEval+from products.posthog_ai.eval_harness.config import SandboxedEvalCase+from products.posthog_ai.eval_harness.harness.context import EvalContext+from products.tasks.backend.facade.agents import CustomPromptSandboxContext++NO_SEND_LOGS = True++DASHBOARD_NAME = "Release health"+ERROR_TRACKING_WIDGET = "error_tracking_list"+LOGS_WIDGET = "logs_list"+SEEDED_ERROR_TRACKING_CONFIG: dict[str, Any] = {"limit": 5, "orderBy": "last_seen", "status": "active"}+++def seed_empty_dashboard(context: CustomPromptSandboxContext) -> dict[str, Any]:+ if Dashboard.objects.filter(team_id=context.team_id, name=DASHBOARD_NAME, deleted=False).exists():+ raise ValueError("Dashboard widget eval requires an unused dashboard name")+ dashboard = Dashboard.objects.create(team_id=context.team_id, name=DASHBOARD_NAME)+ return {"team_id": context.team_id, "dashboard_id": dashboard.id}+++def seed_error_tracking_widget(context: CustomPromptSandboxContext) -> dict[str, Any]:+ seed = seed_empty_dashboard(context)+ config = validate_widget_config(ERROR_TRACKING_WIDGET, SEEDED_ERROR_TRACKING_CONFIG)+ widget = DashboardWidget.objects.for_team(context.team_id).create(+ team_id=context.team_id, widget_type=ERROR_TRACKING_WIDGET, config=config+ )+ DashboardTile.objects.create(+ dashboard_id=seed["dashboard_id"], widget=widget, layouts={"sm": {"x": 0, "y": 0, "w": 6, "h": 5}}+ )+ return {**seed, "widget_id": str(widget.id), "widget_config": config}+++async def eval_dashboard_widgets(ctx: EvalContext) -> None:+ await SandboxedPrivateEval(+ no_send_logs=NO_SEND_LOGS,+ experiment_name="sandboxed-dashboards-widgets-cli",+ offline_suite=offline_suite(+ "dashboards/eval_dashboard_widgets::eval_dashboard_widgets",+ metrics=["saved_widget_state", "widget_first_write_valid"],+ ),+ send_to_posthog=True,+ cases=[+ SandboxedEvalCase(+ name="add_error_tracking_widget",+ prompt=(+ f"Add an error tracking widget to the existing '{DASHBOARD_NAME}' dashboard. "+ "It should list the 15 resolved issues that affected the most users."+ ),+ setup=seed_empty_dashboard,+ expected={+ "saved_widget_state": {+ "widget_type": ERROR_TRACKING_WIDGET,+ "config": {"limit": 15, "orderBy": "users", "status": "resolved"},+ }+ },+ ),+ SandboxedEvalCase(+ name="add_logs_widget",+ prompt=(+ f"Add a logs widget to the existing '{DASHBOARD_NAME}' dashboard. "+ "It should show only error and fatal logs from the 'api' service, oldest first."+ ),+ setup=seed_empty_dashboard,+ expected={+ "saved_widget_state": {+ "widget_type": LOGS_WIDGET,+ "config": {+ "severityLevels": ["error", "fatal"],+ "serviceNames": ["api"],+ "orderBy": "earliest",+ },+ }+ },+ ),+ SandboxedEvalCase(+ name="update_error_tracking_widget",+ prompt=(+ f"On the '{DASHBOARD_NAME}' dashboard, change the error tracking widget to show 20 issues. "+ "Keep its other settings as they are."+ ),+ setup=seed_error_tracking_widget,+ expected={+ "saved_widget_state": {+ "widget_type": ERROR_TRACKING_WIDGET,+ "config": {"limit": 20},+ "preserved": ["orderBy", "status"],+ }+ },+ ),+ SandboxedEvalCase(+ name="resize_error_tracking_widget",+ prompt=(+ f"On the '{DASHBOARD_NAME}' dashboard, make the error tracking widget full width. "+ "Do not change what it shows."+ ),+ setup=seed_error_tracking_widget,+ expected={+ "saved_widget_state": {+ "widget_type": ERROR_TRACKING_WIDGET,+ "preserved": "all",+ "layout_width": 12,+ }+ },+ ),+ ],+ scorers=[SavedWidgetState(), WidgetFirstWriteValid()],+ ctx=ctx,+ )diff --git a/products/dashboards/evals/offline_config.py b/products/dashboards/evals/offline_config.pyindex a6b132274628..f22cfdc4404c 100644--- a/products/dashboards/evals/offline_config.py+++ b/products/dashboards/evals/offline_config.py@@ -10,6 +10,8 @@ _SCORERS: dict[str, OfflineScorer] = {"sql_tiles_follow_dashboard_date_range": OfflineScorer(kind="numeric", version=UUID("01a15a27-d794-74b3-8822-063a74a674fd")),+ "saved_widget_state": OfflineScorer(kind="boolean", version=UUID("01a16268-3277-718d-966d-e22228469dc2")),+ "widget_first_write_valid": OfflineScorer(kind="boolean", version=UUID("01a16268-3643-7628-b863-56b1c091b4ec")),}diff --git a/products/dashboards/evals/scorers.py b/products/dashboards/evals/scorers.pyindex e62aa92263ba..ec0e0a00a6ad 100644--- a/products/dashboards/evals/scorers.py+++ b/products/dashboards/evals/scorers.py@@ -16,6 +16,7 @@ from posthog.hogql_queries.query_runner import get_query_runnerfrom posthog.models.team import Teamfrom products.dashboards.backend.models.dashboard import Dashboard+from products.posthog_ai.eval_harness.log_parser import LogParser, ToolCallfrom products.posthog_ai.eval_harness.scorers.contract import AsyncOnlyScorerMixin, Score, Scorerfrom products.product_analytics.backend.facade.models import Insight@@ -159,3 +160,102 @@ class SQLTilesFollowDashboardDateRange(AsyncOnlyScorerMixin, Scorer):"errors": {str(tile["insight_id"]): tile["error"] for tile in tiles if "error" in tile},},)+++WIDGET_WRITE_TOOLS = frozenset({"dashboard-widgets-batch-add", "dashboard-widgets-batch-update", "dashboard-update"})+WIDGET_CATALOG_TOOL = "dashboard-widget-catalog-list"+++class WidgetFirstWriteValid(Scorer):+ """Compare this score across widget tool schema changes.++ A schema that hides the config keys must not cost the agent a rejected first write.+ """++ def _name(self) -> str:+ return "widget_first_write_valid"++ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:+ if not output or not output.get("raw_log"):+ return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})+ parser = LogParser.cached(output["raw_log"], initial_prompt=output.get("prompt", "") or "")+ calls = parser.get_tool_calls()+ writes = [call for call in calls if _is_widget_write(call)]+ if not writes:+ return Score(name=self._name(), score=0.0, metadata={"reason": "No widget write call"})++ first_success = next((index for index, call in enumerate(writes) if not call.is_error), None)+ catalog_positions = [call.position for call in calls if call.name == WIDGET_CATALOG_TOOL and not call.is_error]+ return Score(+ name=self._name(),+ score=float(not writes[0].is_error),+ metadata={+ "write_attempts": len(writes),+ "failed_writes": sum(call.is_error for call in writes),+ "failed_before_first_success": len(writes) if first_success is None else first_success,+ "write_tools": [call.name for call in writes],+ "catalog_before_first_write": any(position < writes[0].position for position in catalog_positions),+ },+ )+++class SavedWidgetState(AsyncOnlyScorerMixin, Scorer):+ def _name(self) -> str:+ return "saved_widget_state"++ @staticmethod+ def _read_state(team_id: int, dashboard_id: int, widget_type: str) -> list[dict[str, Any]]:+ dashboard = Dashboard.objects.get(team_id=team_id, id=dashboard_id)+ rows = (+ dashboard.tiles.exclude(deleted=True)+ .filter(widget__widget_type=widget_type)+ .values("id", "layouts", "widget_id", "widget__config")+ )+ return [dict(row) for row in rows]++ async def _run_eval_async(+ self, output: dict[str, Any] | None, expected: dict[str, Any] | None = None, **kwargs: Any+ ) -> Score:+ seed = (output or {}).get("seed") or {}+ spec = (expected or {}).get(self._name())+ if not seed.get("team_id") or not seed.get("dashboard_id") or not spec:+ return Score(name=self._name(), score=0.0, metadata={"reason": "Missing seed or expected widget"})++ tiles = await asyncio.to_thread(self._read_state, seed["team_id"], seed["dashboard_id"], spec["widget_type"])+ if len(tiles) != 1:+ return Score(+ name=self._name(),+ score=0.0,+ metadata={"reason": f"Expected one {spec['widget_type']} tile", "tile_count": len(tiles)},+ )++ tile = tiles[0]+ config = tile["widget__config"] or {}+ seeded_config = seed.get("widget_config") or {}+ checks = {+ "requested_config": all(+ _same_value(config.get(key), value) for key, value in spec.get("config", {}).items()+ ),+ "preserved_config": config == seeded_config+ if spec.get("preserved") == "all"+ else all(_same_value(config.get(key), seeded_config.get(key)) for key in spec.get("preserved", [])),+ "same_widget": "widget_id" not in seed or str(tile["widget_id"]) == str(seed["widget_id"]),+ "layout_width": "layout_width" not in spec+ or ((tile["layouts"] or {}).get("sm") or {}).get("w") == spec["layout_width"],+ }+ failed = [name for name, passed in checks.items() if not passed]+ return Score(name=self._name(), score=float(not failed), metadata={"failed_checks": failed, "config": config})+++def _is_widget_write(call: ToolCall) -> bool:+ # dashboard-update also edits dashboard metadata, which does not touch any widget.+ if call.name == "dashboard-update":+ return "tiles" in call.input+ return call.name in WIDGET_WRITE_TOOLS+++def _same_value(actual: Any, wanted: Any) -> bool:+ # The agent may list multi-select values in any order.+ if isinstance(actual, list) and isinstance(wanted, list):+ return sorted(map(str, actual)) == sorted(map(str, wanted))+ return actual == wanteddiff --git a/products/posthog_ai/eval_harness/harness/services.py b/products/posthog_ai/eval_harness/harness/services.pyindex ecfd72ad0746..52915ef2ce85 100644--- a/products/posthog_ai/eval_harness/harness/services.py+++ b/products/posthog_ai/eval_harness/harness/services.py@@ -328,9 +328,14 @@ def start_mcp_server(# legacy notebooks-create / notebooks-retrieve pair, which the two surfaces# being mutually exclusive makes unavoidable — an eval of the legacy tools# needs its own lever, not this one. mcp-exec-skills follows the run's- # skill delivery mode.+ # skill delivery mode. dashboard-widgets gates the widget batch tools that+ # the dashboard widget evals exercise."FEATURE_FLAG_OVERRIDES": json.dumps(- {"revamped-py-notebooks": True, MCP_EXEC_SKILLS_FEATURE_FLAG: exec_skills_enabled}+ {+ "revamped-py-notebooks": True,+ "dashboard-widgets": True,+ MCP_EXEC_SKILLS_FEATURE_FLAG: exec_skills_enabled,+ }),}diff --git a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.pyindex 4bf762a8ce0a..8acfd27a1585 100644--- a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py+++ b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py@@ -92,6 +92,7 @@ def test_start_mcp_server_isolates_the_selected_skill_delivery(assert start.call_args.kwargs.get("readiness_timeout", 30) == (120 if exec_skills_enabled else 30)# The skill delivery switch must not displace the overrides every eval run relies on.assert overrides["revamped-py-notebooks"] is True+ assert overrides["dashboard-widgets"] is Trueif skill_archive_url is None:assert "POSTHOG_MCP_SKILLS_URL" not in envelse:diff --git a/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py b/products/posthog_ai/eval_harness/test/test_saved_object_scorers.pyindex 297d40f2e43f..7592bfd1a716 100644--- a/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py+++ b/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py@@ -14,9 +14,15 @@ from parameterized import parameterizedfrom products.dashboards.backend.models.dashboard import Dashboardfrom products.dashboards.backend.models.dashboard_tile import DashboardTile-from products.dashboards.evals.scorers import SavedDashboardContents, SQLTilesFollowDashboardDateRange+from products.dashboards.evals.scorers import (+ SavedDashboardContents,+ SavedWidgetState,+ SQLTilesFollowDashboardDateRange,+ WidgetFirstWriteValid,+)from products.feature_flags.backend.models.feature_flag import FeatureFlagfrom products.feature_flags.evals.creation_scorers import CreatedFlagConfiguration+from products.posthog_ai.eval_harness.test.test_eval_scorers import _raw_tool_logfrom products.product_analytics.backend.facade.models import Insight@@ -165,3 +171,97 @@ class TestSQLTilesFollowDashboardDateRange(ClickhouseTestMixin, APIBaseTest):)assert score.score == expected_score, score.metadata+++_SEEDED_WIDGET_CONFIG = {"limit": 5, "orderBy": "last_seen", "status": "active", "severityLevels": ["error", "fatal"]}+++ [+ ("requested_change_kept_the_rest", {}, 1.0),+ ("multi_select_in_another_order", {"config": {"severityLevels": ["fatal", "error"]}}, 1.0),+ ("requested_value_missing", {"config": {"limit": 10}}, 0.0),+ ("other_setting_lost", {"config": {"status": "all"}}, 0.0),+ ("widget_recreated", {"widget_id": "new-widget"}, 0.0),+ ("duplicate_widget", {"tile_count": 2}, 0.0),+ ("not_full_width", {"layouts": {"sm": {"w": 6}}}, 0.0),+ ("whole_config_kept", {"preserve_all": True, "config": {"limit": 5}}, 1.0),+ (+ "default_changed_when_all_must_stay",+ {"preserve_all": True, "config": {"limit": 5, "orderDirection": "ASC"}},+ 0.0,+ ),+ ]+)+def test_widget_state_scores_config_identity_and_width(+ _name: str, change: dict[str, Any], expected_score: float+) -> None:+ config = {**_SEEDED_WIDGET_CONFIG, "limit": 20, **change.get("config", {})}+ tile = {+ "id": 1,+ "layouts": change.get("layouts", {"sm": {"w": 12}}),+ "widget_id": change.get("widget_id", "seeded-widget"),+ "widget__config": config,+ }+ seed = {"team_id": 42, "dashboard_id": 7, "widget_id": "seeded-widget", "widget_config": _SEEDED_WIDGET_CONFIG}+ spec: dict[str, Any] = {"widget_type": "error_tracking_list", "layout_width": 12}+ if change.get("preserve_all"):+ spec["preserved"] = "all"+ else:+ spec.update({"config": {"limit": 20, "severityLevels": ["error", "fatal"]}, "preserved": ["orderBy", "status"]})+ with patch.object(SavedWidgetState, "_read_state", return_value=[tile] * change.get("tile_count", 1)):+ score = asyncio.run(SavedWidgetState().eval_async({"seed": seed}, {"saved_widget_state": spec}))++ assert score.score == expected_score, score.metadata+++def _exec(command: str, status: str = "completed") -> tuple[str, dict[str, str], str, str]:+ return ("mcp__posthog__exec", {"command": command}, "ok", status)+++ [+ (+ "first_write_accepted_after_reading_the_catalog",+ [+ _exec("info dashboard-widgets-batch-add"),+ _exec("call dashboard-widget-catalog-list {}"),+ _exec('call dashboard-widgets-batch-add {"id": 7}'),+ ],+ 1.0,+ {"write_attempts": 1, "failed_before_first_success": 0, "catalog_before_first_write": True},+ ),+ (+ "first_write_rejected_then_fixed",+ [+ _exec('call dashboard-widgets-batch-add {"id": 7}', "failed"),+ _exec('call dashboard-widgets-batch-add {"id": 7}'),+ ],+ 0.0,+ {"write_attempts": 2, "failed_before_first_success": 1, "catalog_before_first_write": False},+ ),+ (+ "reads_do_not_count_as_writes",+ [_exec("call dashboard-get {}"), _exec('call dashboard-update {"id": 7, "tiles": []}')],+ 1.0,+ {"write_attempts": 1, "write_tools": ["dashboard-update"]},+ ),+ (+ "dashboard_metadata_update_is_not_a_widget_write",+ [+ _exec('call dashboard-update {"id": 7, "name": "Renamed"}'),+ _exec('call dashboard-widgets-batch-add {"id": 7}', "failed"),+ ],+ 0.0,+ {"write_attempts": 1, "write_tools": ["dashboard-widgets-batch-add"]},+ ),+ ("no_widget_write", [_exec("call dashboard-get {}")], 0.0, {}),+ ]+)+def test_widget_first_write_valid_scores_the_first_widget_write(+ _name: str, calls: list[tuple[Any, ...]], expected_score: float, expected_metadata: dict[str, Any]+) -> None:+ score = WidgetFirstWriteValid().eval({"raw_log": _raw_tool_log(calls), "prompt": ""})++ assert score.score == expected_score, score.metadata+ assert {key: score.metadata[key] for key in expected_metadata} == expected_metadata