Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(dashboards): add an agent eval for dashboard widget tools (#120220)

Thomas Obermüllerce345fca75be
diff --git a/.agents/skills/manage-dashboard-widgets/references/mcp.md b/.agents/skills/manage-dashboard-widgets/references/mcp.md
index 0cd08bbf92b1..6265633a4c1a 100644
--- a/.agents/skills/manage-dashboard-widgets/references/mcp.md
+++ b/.agents/skills/manage-dashboard-widgets/references/mcp.md
@@ -55,3 +55,4 @@ UI multi-select add uses `POST .../widgets/batch/` (not PATCH dashboard `tiles[]
1. Regenerate OpenAPI (`hogli build:openapi`) — updates MCP tool schemas that embed supported type lists
2. Run `hogli test services/mcp/tests/tools/dashboards.integration.test.ts`
3. Update unit schema snapshots if batch-add help text lists types: `hogli test services/mcp/tests/unit/tool-schema-snapshots.test.ts`
+4. To check that agents can configure the new type, add a case to `products/dashboards/evals/eval_dashboard_widgets.py` and run `hogli evals eval_dashboard_widgets` (see the `writing-evals` skill)
diff --git a/products/dashboards/evals/eval_dashboard_widgets.py b/products/dashboards/evals/eval_dashboard_widgets.py
new file mode 100644
index 000000000000..45cf2a69aa55
--- /dev/null
+++ b/products/dashboards/evals/eval_dashboard_widgets.py
@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from typing import Any
+
+from products.dashboards.backend.models.dashboard import Dashboard
+from products.dashboards.backend.models.dashboard_tile import DashboardTile
+from products.dashboards.backend.models.dashboard_widget import DashboardWidget
+from products.dashboards.backend.widget_specs.registry import validate_widget_config
+from products.dashboards.evals.offline_config import offline_suite
+from products.dashboards.evals.scorers import SavedWidgetState, WidgetFirstWriteValid
+from products.posthog_ai.eval_harness.base import SandboxedPrivateEval
+from products.posthog_ai.eval_harness.config import SandboxedEvalCase
+from products.posthog_ai.eval_harness.harness.context import EvalContext
+from products.tasks.backend.facade.agents import CustomPromptSandboxContext
+
+NO_SEND_LOGS = True
+
+DASHBOARD_NAME = "Release health"
+ERROR_TRACKING_WIDGET = "error_tracking_list"
+LOGS_WIDGET = "logs_list"
+SEEDED_ERROR_TRACKING_CONFIG: dict[str, Any] = {"limit": 5, "orderBy": "last_seen", "status": "active"}
+
+
+def seed_empty_dashboard(context: CustomPromptSandboxContext) -> dict[str, Any]:
+ if Dashboard.objects.filter(team_id=context.team_id, name=DASHBOARD_NAME, deleted=False).exists():
+ raise ValueError("Dashboard widget eval requires an unused dashboard name")
+ dashboard = Dashboard.objects.create(team_id=context.team_id, name=DASHBOARD_NAME)
+ return {"team_id": context.team_id, "dashboard_id": dashboard.id}
+
+
+def seed_error_tracking_widget(context: CustomPromptSandboxContext) -> dict[str, Any]:
+ seed = seed_empty_dashboard(context)
+ config = validate_widget_config(ERROR_TRACKING_WIDGET, SEEDED_ERROR_TRACKING_CONFIG)
+ widget = DashboardWidget.objects.for_team(context.team_id).create(
+ team_id=context.team_id, widget_type=ERROR_TRACKING_WIDGET, config=config
+ )
+ DashboardTile.objects.create(
+ dashboard_id=seed["dashboard_id"], widget=widget, layouts={"sm": {"x": 0, "y": 0, "w": 6, "h": 5}}
+ )
+ return {**seed, "widget_id": str(widget.id), "widget_config": config}
+
+
+async def eval_dashboard_widgets(ctx: EvalContext) -> None:
+ await SandboxedPrivateEval(
+ no_send_logs=NO_SEND_LOGS,
+ experiment_name="sandboxed-dashboards-widgets-cli",
+ offline_suite=offline_suite(
+ "dashboards/eval_dashboard_widgets::eval_dashboard_widgets",
+ metrics=["saved_widget_state", "widget_first_write_valid"],
+ ),
+ send_to_posthog=True,
+ cases=[
+ SandboxedEvalCase(
+ name="add_error_tracking_widget",
+ prompt=(
+ f"Add an error tracking widget to the existing '{DASHBOARD_NAME}' dashboard. "
+ "It should list the 15 resolved issues that affected the most users."
+ ),
+ setup=seed_empty_dashboard,
+ expected={
+ "saved_widget_state": {
+ "widget_type": ERROR_TRACKING_WIDGET,
+ "config": {"limit": 15, "orderBy": "users", "status": "resolved"},
+ }
+ },
+ ),
+ SandboxedEvalCase(
+ name="add_logs_widget",
+ prompt=(
+ f"Add a logs widget to the existing '{DASHBOARD_NAME}' dashboard. "
+ "It should show only error and fatal logs from the 'api' service, oldest first."
+ ),
+ setup=seed_empty_dashboard,
+ expected={
+ "saved_widget_state": {
+ "widget_type": LOGS_WIDGET,
+ "config": {
+ "severityLevels": ["error", "fatal"],
+ "serviceNames": ["api"],
+ "orderBy": "earliest",
+ },
+ }
+ },
+ ),
+ SandboxedEvalCase(
+ name="update_error_tracking_widget",
+ prompt=(
+ f"On the '{DASHBOARD_NAME}' dashboard, change the error tracking widget to show 20 issues. "
+ "Keep its other settings as they are."
+ ),
+ setup=seed_error_tracking_widget,
+ expected={
+ "saved_widget_state": {
+ "widget_type": ERROR_TRACKING_WIDGET,
+ "config": {"limit": 20},
+ "preserved": ["orderBy", "status"],
+ }
+ },
+ ),
+ SandboxedEvalCase(
+ name="resize_error_tracking_widget",
+ prompt=(
+ f"On the '{DASHBOARD_NAME}' dashboard, make the error tracking widget full width. "
+ "Do not change what it shows."
+ ),
+ setup=seed_error_tracking_widget,
+ expected={
+ "saved_widget_state": {
+ "widget_type": ERROR_TRACKING_WIDGET,
+ "preserved": "all",
+ "layout_width": 12,
+ }
+ },
+ ),
+ ],
+ scorers=[SavedWidgetState(), WidgetFirstWriteValid()],
+ ctx=ctx,
+ )
diff --git a/products/dashboards/evals/offline_config.py b/products/dashboards/evals/offline_config.py
index a6b132274628..f22cfdc4404c 100644
--- a/products/dashboards/evals/offline_config.py
+++ b/products/dashboards/evals/offline_config.py
@@ -10,6 +10,8 @@ _SCORERS: dict[str, OfflineScorer] = {
"sql_tiles_follow_dashboard_date_range": OfflineScorer(
kind="numeric", version=UUID("01a15a27-d794-74b3-8822-063a74a674fd")
),
+ "saved_widget_state": OfflineScorer(kind="boolean", version=UUID("01a16268-3277-718d-966d-e22228469dc2")),
+ "widget_first_write_valid": OfflineScorer(kind="boolean", version=UUID("01a16268-3643-7628-b863-56b1c091b4ec")),
}
diff --git a/products/dashboards/evals/scorers.py b/products/dashboards/evals/scorers.py
index e62aa92263ba..ec0e0a00a6ad 100644
--- a/products/dashboards/evals/scorers.py
+++ b/products/dashboards/evals/scorers.py
@@ -16,6 +16,7 @@ from posthog.hogql_queries.query_runner import get_query_runner
from posthog.models.team import Team
from products.dashboards.backend.models.dashboard import Dashboard
+from products.posthog_ai.eval_harness.log_parser import LogParser, ToolCall
from products.posthog_ai.eval_harness.scorers.contract import AsyncOnlyScorerMixin, Score, Scorer
from products.product_analytics.backend.facade.models import Insight
@@ -159,3 +160,102 @@ class SQLTilesFollowDashboardDateRange(AsyncOnlyScorerMixin, Scorer):
"errors": {str(tile["insight_id"]): tile["error"] for tile in tiles if "error" in tile},
},
)
+
+
+WIDGET_WRITE_TOOLS = frozenset({"dashboard-widgets-batch-add", "dashboard-widgets-batch-update", "dashboard-update"})
+WIDGET_CATALOG_TOOL = "dashboard-widget-catalog-list"
+
+
+class WidgetFirstWriteValid(Scorer):
+ """Compare this score across widget tool schema changes.
+
+ A schema that hides the config keys must not cost the agent a rejected first write.
+ """
+
+ def _name(self) -> str:
+ return "widget_first_write_valid"
+
+ def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:
+ if not output or not output.get("raw_log"):
+ return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})
+ parser = LogParser.cached(output["raw_log"], initial_prompt=output.get("prompt", "") or "")
+ calls = parser.get_tool_calls()
+ writes = [call for call in calls if _is_widget_write(call)]
+ if not writes:
+ return Score(name=self._name(), score=0.0, metadata={"reason": "No widget write call"})
+
+ first_success = next((index for index, call in enumerate(writes) if not call.is_error), None)
+ catalog_positions = [call.position for call in calls if call.name == WIDGET_CATALOG_TOOL and not call.is_error]
+ return Score(
+ name=self._name(),
+ score=float(not writes[0].is_error),
+ metadata={
+ "write_attempts": len(writes),
+ "failed_writes": sum(call.is_error for call in writes),
+ "failed_before_first_success": len(writes) if first_success is None else first_success,
+ "write_tools": [call.name for call in writes],
+ "catalog_before_first_write": any(position < writes[0].position for position in catalog_positions),
+ },
+ )
+
+
+class SavedWidgetState(AsyncOnlyScorerMixin, Scorer):
+ def _name(self) -> str:
+ return "saved_widget_state"
+
+ @staticmethod
+ def _read_state(team_id: int, dashboard_id: int, widget_type: str) -> list[dict[str, Any]]:
+ dashboard = Dashboard.objects.get(team_id=team_id, id=dashboard_id)
+ rows = (
+ dashboard.tiles.exclude(deleted=True)
+ .filter(widget__widget_type=widget_type)
+ .values("id", "layouts", "widget_id", "widget__config")
+ )
+ return [dict(row) for row in rows]
+
+ async def _run_eval_async(
+ self, output: dict[str, Any] | None, expected: dict[str, Any] | None = None, **kwargs: Any
+ ) -> Score:
+ seed = (output or {}).get("seed") or {}
+ spec = (expected or {}).get(self._name())
+ if not seed.get("team_id") or not seed.get("dashboard_id") or not spec:
+ return Score(name=self._name(), score=0.0, metadata={"reason": "Missing seed or expected widget"})
+
+ tiles = await asyncio.to_thread(self._read_state, seed["team_id"], seed["dashboard_id"], spec["widget_type"])
+ if len(tiles) != 1:
+ return Score(
+ name=self._name(),
+ score=0.0,
+ metadata={"reason": f"Expected one {spec['widget_type']} tile", "tile_count": len(tiles)},
+ )
+
+ tile = tiles[0]
+ config = tile["widget__config"] or {}
+ seeded_config = seed.get("widget_config") or {}
+ checks = {
+ "requested_config": all(
+ _same_value(config.get(key), value) for key, value in spec.get("config", {}).items()
+ ),
+ "preserved_config": config == seeded_config
+ if spec.get("preserved") == "all"
+ else all(_same_value(config.get(key), seeded_config.get(key)) for key in spec.get("preserved", [])),
+ "same_widget": "widget_id" not in seed or str(tile["widget_id"]) == str(seed["widget_id"]),
+ "layout_width": "layout_width" not in spec
+ or ((tile["layouts"] or {}).get("sm") or {}).get("w") == spec["layout_width"],
+ }
+ failed = [name for name, passed in checks.items() if not passed]
+ return Score(name=self._name(), score=float(not failed), metadata={"failed_checks": failed, "config": config})
+
+
+def _is_widget_write(call: ToolCall) -> bool:
+ # dashboard-update also edits dashboard metadata, which does not touch any widget.
+ if call.name == "dashboard-update":
+ return "tiles" in call.input
+ return call.name in WIDGET_WRITE_TOOLS
+
+
+def _same_value(actual: Any, wanted: Any) -> bool:
+ # The agent may list multi-select values in any order.
+ if isinstance(actual, list) and isinstance(wanted, list):
+ return sorted(map(str, actual)) == sorted(map(str, wanted))
+ return actual == wanted
diff --git a/products/posthog_ai/eval_harness/harness/services.py b/products/posthog_ai/eval_harness/harness/services.py
index ecfd72ad0746..52915ef2ce85 100644
--- a/products/posthog_ai/eval_harness/harness/services.py
+++ b/products/posthog_ai/eval_harness/harness/services.py
@@ -328,9 +328,14 @@ def start_mcp_server(
# legacy notebooks-create / notebooks-retrieve pair, which the two surfaces
# being mutually exclusive makes unavoidable — an eval of the legacy tools
# needs its own lever, not this one. mcp-exec-skills follows the run's
- # skill delivery mode.
+ # skill delivery mode. dashboard-widgets gates the widget batch tools that
+ # the dashboard widget evals exercise.
"FEATURE_FLAG_OVERRIDES": json.dumps(
- {"revamped-py-notebooks": True, MCP_EXEC_SKILLS_FEATURE_FLAG: exec_skills_enabled}
+ {
+ "revamped-py-notebooks": True,
+ "dashboard-widgets": True,
+ MCP_EXEC_SKILLS_FEATURE_FLAG: exec_skills_enabled,
+ }
),
}
diff --git a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py
index 4bf762a8ce0a..8acfd27a1585 100644
--- a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py
+++ b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py
@@ -92,6 +92,7 @@ def test_start_mcp_server_isolates_the_selected_skill_delivery(
assert start.call_args.kwargs.get("readiness_timeout", 30) == (120 if exec_skills_enabled else 30)
# The skill delivery switch must not displace the overrides every eval run relies on.
assert overrides["revamped-py-notebooks"] is True
+ assert overrides["dashboard-widgets"] is True
if skill_archive_url is None:
assert "POSTHOG_MCP_SKILLS_URL" not in env
else:
diff --git a/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py b/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py
index 297d40f2e43f..7592bfd1a716 100644
--- a/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py
+++ b/products/posthog_ai/eval_harness/test/test_saved_object_scorers.py
@@ -14,9 +14,15 @@ from parameterized import parameterized
from products.dashboards.backend.models.dashboard import Dashboard
from products.dashboards.backend.models.dashboard_tile import DashboardTile
-from products.dashboards.evals.scorers import SavedDashboardContents, SQLTilesFollowDashboardDateRange
+from products.dashboards.evals.scorers import (
+ SavedDashboardContents,
+ SavedWidgetState,
+ SQLTilesFollowDashboardDateRange,
+ WidgetFirstWriteValid,
+)
from products.feature_flags.backend.models.feature_flag import FeatureFlag
from products.feature_flags.evals.creation_scorers import CreatedFlagConfiguration
+from products.posthog_ai.eval_harness.test.test_eval_scorers import _raw_tool_log
from products.product_analytics.backend.facade.models import Insight
@@ -165,3 +171,97 @@ class TestSQLTilesFollowDashboardDateRange(ClickhouseTestMixin, APIBaseTest):
)
assert score.score == expected_score, score.metadata
+
+
+_SEEDED_WIDGET_CONFIG = {"limit": 5, "orderBy": "last_seen", "status": "active", "severityLevels": ["error", "fatal"]}
+
+
+ [
+ ("requested_change_kept_the_rest", {}, 1.0),
+ ("multi_select_in_another_order", {"config": {"severityLevels": ["fatal", "error"]}}, 1.0),
+ ("requested_value_missing", {"config": {"limit": 10}}, 0.0),
+ ("other_setting_lost", {"config": {"status": "all"}}, 0.0),
+ ("widget_recreated", {"widget_id": "new-widget"}, 0.0),
+ ("duplicate_widget", {"tile_count": 2}, 0.0),
+ ("not_full_width", {"layouts": {"sm": {"w": 6}}}, 0.0),
+ ("whole_config_kept", {"preserve_all": True, "config": {"limit": 5}}, 1.0),
+ (
+ "default_changed_when_all_must_stay",
+ {"preserve_all": True, "config": {"limit": 5, "orderDirection": "ASC"}},
+ 0.0,
+ ),
+ ]
+)
+def test_widget_state_scores_config_identity_and_width(
+ _name: str, change: dict[str, Any], expected_score: float
+) -> None:
+ config = {**_SEEDED_WIDGET_CONFIG, "limit": 20, **change.get("config", {})}
+ tile = {
+ "id": 1,
+ "layouts": change.get("layouts", {"sm": {"w": 12}}),
+ "widget_id": change.get("widget_id", "seeded-widget"),
+ "widget__config": config,
+ }
+ seed = {"team_id": 42, "dashboard_id": 7, "widget_id": "seeded-widget", "widget_config": _SEEDED_WIDGET_CONFIG}
+ spec: dict[str, Any] = {"widget_type": "error_tracking_list", "layout_width": 12}
+ if change.get("preserve_all"):
+ spec["preserved"] = "all"
+ else:
+ spec.update({"config": {"limit": 20, "severityLevels": ["error", "fatal"]}, "preserved": ["orderBy", "status"]})
+ with patch.object(SavedWidgetState, "_read_state", return_value=[tile] * change.get("tile_count", 1)):
+ score = asyncio.run(SavedWidgetState().eval_async({"seed": seed}, {"saved_widget_state": spec}))
+
+ assert score.score == expected_score, score.metadata
+
+
+def _exec(command: str, status: str = "completed") -> tuple[str, dict[str, str], str, str]:
+ return ("mcp__posthog__exec", {"command": command}, "ok", status)
+
+
+ [
+ (
+ "first_write_accepted_after_reading_the_catalog",
+ [
+ _exec("info dashboard-widgets-batch-add"),
+ _exec("call dashboard-widget-catalog-list {}"),
+ _exec('call dashboard-widgets-batch-add {"id": 7}'),
+ ],
+ 1.0,
+ {"write_attempts": 1, "failed_before_first_success": 0, "catalog_before_first_write": True},
+ ),
+ (
+ "first_write_rejected_then_fixed",
+ [
+ _exec('call dashboard-widgets-batch-add {"id": 7}', "failed"),
+ _exec('call dashboard-widgets-batch-add {"id": 7}'),
+ ],
+ 0.0,
+ {"write_attempts": 2, "failed_before_first_success": 1, "catalog_before_first_write": False},
+ ),
+ (
+ "reads_do_not_count_as_writes",
+ [_exec("call dashboard-get {}"), _exec('call dashboard-update {"id": 7, "tiles": []}')],
+ 1.0,
+ {"write_attempts": 1, "write_tools": ["dashboard-update"]},
+ ),
+ (
+ "dashboard_metadata_update_is_not_a_widget_write",
+ [
+ _exec('call dashboard-update {"id": 7, "name": "Renamed"}'),
+ _exec('call dashboard-widgets-batch-add {"id": 7}', "failed"),
+ ],
+ 0.0,
+ {"write_attempts": 1, "write_tools": ["dashboard-widgets-batch-add"]},
+ ),
+ ("no_widget_write", [_exec("call dashboard-get {}")], 0.0, {}),
+ ]
+)
+def test_widget_first_write_valid_scores_the_first_widget_write(
+ _name: str, calls: list[tuple[Any, ...]], expected_score: float, expected_metadata: dict[str, Any]
+) -> None:
+ score = WidgetFirstWriteValid().eval({"raw_log": _raw_tool_log(calls), "prompt": ""})
+
+ assert score.score == expected_score, score.metadata
+ assert {key: score.metadata[key] for key in expected_metadata} == expected_metadata