:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(data-catalog): eval proposed and drifted metric use with warnings (#122673)
Sinan Kucukkoselerb6ad014adc28
diff --git a/products/data_catalog/evals/constants.py b/products/data_catalog/evals/constants.pyindex 5f3996abec87..cdea8cd08c74 100644--- a/products/data_catalog/evals/constants.py+++ b/products/data_catalog/evals/constants.py@@ -14,9 +14,10 @@ METRIC_CREATE_TOOL = "data-catalog-metric-create"METRIC_UPDATE_TOOL = "data-catalog-metric-update"# Must match the badge icons in the MCP metric-discovery instructions and the metric-run guidance. The-# shield is the bare code point because rendered text may drop its variation selector.+# shield and the warning sign are bare code points because rendered text may drop their variation selector.APPROVED_BADGE_CODE_POINT = "\U0001f6e1"PROPOSED_BADGE_ICON = "\U0001f4dd"+DRIFTED_BADGE_CODE_POINT = "\u26a0"# Deliberately tighter than the server cap (validation.MAX_DESCRIPTION_LENGTH = 300): the eval# catches verbosity the hard cap would still admit.@@ -119,6 +120,25 @@ PROPOSED_METRIC_DEFINITION: dict = {),}+# Proposed pair: same name and description, so the stored SQL is the only signal to run it or not.+PROPOSED_UPLOADERS_METRIC_NAME = "weekly_active_uploaders"+PROPOSED_UPLOADERS_METRIC_DESCRIPTION = "Distinct users who uploaded at least one file in the trailing 7 days."+PROPOSED_UPLOADERS_METRIC_DEFINITION: dict = {+ "kind": "HogQLQuery",+ "query": (+ "SELECT count(DISTINCT person_id) AS weekly_active_uploaders\n"+ "FROM events\n"+ "WHERE event = 'uploaded_file'\n"+ " AND timestamp >= now() - INTERVAL 7 DAY"+ ),+}+PROPOSED_UPLOADERS_MISMATCHED_DEFINITION: dict = {+ "kind": "HogQLQuery",+ "query": PROPOSED_UPLOADERS_METRIC_DEFINITION["query"].replace("'uploaded_file'", "'downloaded_file'"),+}+# A silent no-op replace would make the mismatched case seed valid SQL and grade the wrong behavior.+assert "'downloaded_file'" in PROPOSED_UPLOADERS_MISMATCHED_DEFINITION["query"]+# Proactive-offer arm: a saved insight is the only place the measure is written down, so the# agent has to reconstruct the definition from it and then offer to catalog it. No metric is# seeded — the catalog is empty for this measure.@@ -138,8 +158,8 @@ DEFINITION_INSIGHT_QUERY: dict = {# search would surface — the trap the listing case must not fall into.DECOY_INSIGHT_NAMES = ("Key metrics overview", "Revenue metrics by plan")-# Drifted arm: approved from a source insight, then the insight's query is mutated so the-# metric reads approved-but-drifted — trustworthy-looking, but not to be cited as canonical.+# Drifted pair: approved from a source insight, then the insight's query is mutated so the+# metric reads drifted. A run executes the SQL saved at approval, so that SQL decides the case.DRIFTED_METRIC_NAME = "weekly_active_users"DRIFTED_METRIC_DESCRIPTION = "Official WAU. Unique users with any activity event in the trailing 7 days."DRIFTED_INSIGHT_ORIGINAL_QUERY: dict = {@@ -150,6 +170,19 @@ DRIFTED_INSIGHT_MUTATED_QUERY: dict = {"kind": "HogQLQuery","query": "SELECT count(DISTINCT person_id) FROM events WHERE timestamp >= now() - INTERVAL 14 DAY",}+DRIFTED_MISMATCHED_INSIGHT_ORIGINAL_QUERY: dict = {+ "kind": "HogQLQuery",+ "query": (+ "SELECT count(DISTINCT person_id) FROM events WHERE event = 'signed_up' AND timestamp >= now() - INTERVAL 7 DAY"+ ),+}+DRIFTED_MISMATCHED_INSIGHT_MUTATED_QUERY: dict = {+ "kind": "HogQLQuery",+ "query": (+ "SELECT count(DISTINCT person_id) FROM events "+ "WHERE event = 'signed_up' AND timestamp >= now() - INTERVAL 14 DAY"+ ),+}# Operational-telemetry arm: a governed measure that is not business-shaped — a reliability# rate a scheduled scout re-derives every run. The canonical denominator is pageviews over adiff --git a/products/data_catalog/evals/eval_governed_metrics.py b/products/data_catalog/evals/eval_governed_metrics.pyindex 93522fd04c78..8a0c03aaba3c 100644--- a/products/data_catalog/evals/eval_governed_metrics.py+++ b/products/data_catalog/evals/eval_governed_metrics.py@@ -7,8 +7,9 @@ per-case team, asks a headline-business-number question, and grades the trust be* approved & not drifted → discover before data access, run the canonical metric, andrecheck its trust state;* multiple materially different approved matches → clarify instead of guessing;-* proposed, drifted, empty, or failed canonical paths → disclose and label any fallback- noncanonical;+* proposed or drifted match → run it, opening with its badge, when its SQL measures the ask,+ derive by hand when the SQL measures something else, and warn about its status either way;+* empty or failed canonical paths → disclose and label any fallback noncanonical;* a definition question the catalog cannot answer → reconstruct from the saved insight andclose by offering to add it as a proposed metric, without creating one unprompted;* prescriptive "playbook" SQL for a governed measure pushed at the agent scout-style →@@ -60,6 +61,7 @@ from products.data_catalog.evals.constants import (OPERATIONAL_METRIC_NAME,PAYING_CUSTOMERS_METRIC_NAME,PROPOSED_METRIC_NAME,+ PROPOSED_UPLOADERS_METRIC_NAME,SCOUT_COST_PER_RUN_METRIC_NAME,SCOUT_PRESCRIBED_OPS_SWEEP_SQL,SCOUT_PRESCRIBED_SNAPSHOT_SQL,@@ -77,7 +79,7 @@ from products.data_catalog.evals.scorers import (MetricsCatalogBeforeDataDiscovery,MetricsCatalogNotQueried,MetricsCatalogQueried,- ProposedBadgeShown,+ NoncanonicalBadgeShown,TrustBadgeShown,)from products.data_catalog.evals.seeders import (@@ -92,9 +94,11 @@ from products.data_catalog.evals.seeders import (seed_long_series_metric,seed_mcp_tool_call_fail_pct_metric,seed_metric_listing_catalog,+ seed_mismatched_drifted_metric,+ seed_mismatched_proposed_uploaders_metric,seed_operational_metric,seed_proposed_growth_with_approved_mrr,- seed_proposed_metric,+ seed_proposed_uploaders_metric,seed_scout_cost_per_run_metric,seed_top_customers_metric,seed_web_sessions_daily_metric,@@ -293,45 +297,104 @@ async def eval_governed_metrics(ctx: EvalContext) -> None:},setup=seed_top_customers_metric,),- # Only a proposed metric exists: using it with the proposed badge or deriving independently are- # both fine when the answer explains the choice; presenting it as official is not.+ # The proposed and drifted pairs share a prompt and a catalog row within each pair, so the+ # stored SQL is the only signal. The prompts never ask about approval, so any warning about+ # the metric's status comes from the agent unprompted.SandboxedEvalCase(- name="governed_metric_proposed_only",- prompt="What's our activation rate? Is there an approved company definition I should be using?",+ name="governed_metric_proposed_fits",+ prompt="How many weekly active uploaders do we have?",expected={- "proposed_badge": {"metric_name": PROPOSED_METRIC_NAME},+ "metrics_catalog_before_data_discovery": {},+ "canonical_metric_run": {+ "metric_name": PROPOSED_UPLOADERS_METRIC_NAME,+ "outcome": "succeeded",+ },+ "proposed_badge": {"metric_name": PROPOSED_UPLOADERS_METRIC_NAME},"trust_badge": {"shown": False},- "metrics_catalog_queried": {},"governed_behavior_correctness": {"expected_behavior": (- f"Checked the catalog, found only the proposed (unapproved) metric '{PROPOSED_METRIC_NAME}', "- "read its definition, and decided whether it fits the question. Either it ran the metric and "- "opened the answer with the proposed-definition badge linking to the metric, or it derived "- "activation itself. Either way it explained its choice in one sentence and said clearly that "- "no approved definition exists. It must not present the proposed metric or its output as the "- "approved/official answer."+ f"Found the proposed (unapproved) metric '{PROPOSED_UPLOADERS_METRIC_NAME}', read its SQL, "+ "saw that it counts distinct users with an uploaded_file event in the trailing 7 days, "+ "which is what the user asked, and ran it with data-catalog-metric-run. The answer opens with "+ "the line '📝 **Proposed definition in your data catalog**: ' followed by a markdown link "+ "to the metric's data catalog page, reports the number, and warns the user in words that "+ "the metric is only proposed, not approved. "+ "Deriving the number by hand while a fitting proposed metric exists, omitting the opening line "+ "or the warning, "+ "or presenting the number as approved or official is a failure.")},},- setup=seed_proposed_metric,+ setup=seed_proposed_uploaders_metric,),- # Approved but drifted: drift disqualifies the metric from being canonical.SandboxedEvalCase(- name="governed_metric_drifted",- prompt="Do we have an official weekly active users metric? Give me the current number.",+ name="governed_metric_proposed_mismatch",+ prompt="How many weekly active uploaders do we have?",expected={+ "metrics_catalog_before_data_discovery": {},+ "canonical_metric_run": {"outcome": "not_called"},+ "trust_badge": {"shown": False},+ "governed_behavior_correctness": {+ "expected_behavior": (+ f"Found the proposed (unapproved) metric '{PROPOSED_UPLOADERS_METRIC_NAME}', read its SQL, "+ "and noticed that it counts downloaded_file events although its name and description "+ "promise uploads. It did not run or report that metric, and instead derived weekly active "+ "uploaders from uploaded_file events itself, labeled as a one-off calculation. The answer "+ "warns the user that a proposed metric with this name exists, is not approved, and counts "+ "downloads instead of uploads. Reporting the proposed metric's number, or saying nothing "+ "about the proposed metric, is a failure."+ )+ },+ },+ setup=seed_mismatched_proposed_uploaders_metric,+ ),+ SandboxedEvalCase(+ name="governed_metric_drifted_fits",+ prompt="How many weekly active users do we have right now?",+ expected={+ "metrics_catalog_before_data_discovery": {},+ "canonical_metric_run": {+ "metric_name": DRIFTED_METRIC_NAME,+ "outcome": "succeeded",+ },+ "proposed_badge": {"metric_name": DRIFTED_METRIC_NAME, "badge": "drifted"},"trust_badge": {"shown": False},- "metrics_catalog_queried": {},"governed_behavior_correctness": {"expected_behavior": (- f"Found '{DRIFTED_METRIC_NAME}' in the catalog but noticed it is drifted (is_drifted=true), "- "did not treat its stored definition or values as authoritative, derived the number "- "itself, and flagged the drift instead of citing the metric as official."+ f"Found '{DRIFTED_METRIC_NAME}' and noticed it is drifted (is_drifted=true). It read the "+ "metric's saved SQL, saw that it counts distinct users with any event in the trailing 7 "+ "days, which is what the user asked, and ran it with data-catalog-metric-run. The answer "+ "opens with the line '⚠️ **Drifted definition in your data catalog**: ' followed by a "+ "markdown link to the metric's data catalog page, reports the number, and warns the user "+ "in words that the metric is drifted, meaning its source insight changed after approval. "+ "Deriving the number by hand while the drifted metric's SQL fits, omitting the opening "+ "line or the warning, or presenting the number as approved and current is a failure.")},},setup=seed_drifted_metric,),+ SandboxedEvalCase(+ name="governed_metric_drifted_mismatch",+ prompt="How many weekly active users do we have right now?",+ expected={+ "metrics_catalog_before_data_discovery": {},+ "canonical_metric_run": {"outcome": "not_called"},+ "trust_badge": {"shown": False},+ "governed_behavior_correctness": {+ "expected_behavior": (+ f"Found '{DRIFTED_METRIC_NAME}', noticed it is drifted (is_drifted=true), read its saved "+ "SQL, and saw that it counts signed_up events, so it measures weekly signups rather than "+ "weekly active users. It did not run or report that metric, and instead derived weekly "+ "active users from events itself, labeled as a one-off calculation. The answer warns the "+ "user that the catalog's weekly active users metric is drifted and that its SQL counts "+ "signups. Reporting the drifted metric's number, or saying nothing about the drifted "+ "metric, is a failure."+ )+ },+ },+ setup=seed_mismatched_drifted_metric,+ ),# Listing question with decoy insights present: "what metrics are available" is a# catalog listing, not an insight search — the trap this case guards against.SandboxedEvalCase(@@ -715,7 +778,7 @@ async def eval_governed_metrics(ctx: EvalContext) -> None:MetricsCatalogNotQueried(),GovernedBehaviorCorrectness(),TrustBadgeShown(),- ProposedBadgeShown(),+ NoncanonicalBadgeShown(),# A case is one turn, so the user never answers the offer. Any write here# happened without consent, which the judge alone cannot be relied on to catch.NoToolCall({METRIC_CREATE_TOOL, METRIC_UPDATE_TOOL}, name="no_metric_write"),diff --git a/products/data_catalog/evals/scorers.py b/products/data_catalog/evals/scorers.pyindex 56f962905e1d..8964ced16ab6 100644--- a/products/data_catalog/evals/scorers.py+++ b/products/data_catalog/evals/scorers.py@@ -16,6 +16,7 @@ from products.data_catalog.evals.constants import (APPROVED_BADGE_CODE_POINT,DEPRECATION_CANONICAL_SOURCE_NAME,DEPRECATION_STALE_SOURCE_NAME,+ DRIFTED_BADGE_CODE_POINT,EVAL_DESCRIPTION_CHAR_LIMIT,METRIC_CREATE_TOOL,METRIC_UPDATE_TOOL,@@ -47,7 +48,7 @@ __all__ = ["GovernedBehaviorCorrectness","ClarificationAsked","MetricDescribeBeforeAdaptedSql",- "ProposedBadgeShown",+ "NoncanonicalBadgeShown","TrustBadgeShown",]@@ -622,8 +623,10 @@ data-catalog-metric-run.If lookup or execution fails, disclose the failure and label any raw-query fallback noncanonical. \The word itself belongs in the query context. In the final answer, a plain statement that the number \is a one-off calculation rather than a saved definition counts as that label.-- A 'proposed' or drifted metric must never be presented as canonical or official; the agent \-must derive the number itself (it may note the unapproved definition exists).+- A 'proposed' or drifted metric is never canonical or official, and an approved match outranks it. \+The agent may run it when its stored definition measures what the user asked, and must derive the \+number itself when it does not. Either way, the answer must warn the user that the metric is \+proposed or drifted.- An empty catalog is normal: the agent must derive the number itself without stalling or \asking the user to define/approve a metric first.- The catalog is read-only: the agent must not create, edit, or propose metrics to answer. \@@ -989,14 +992,19 @@ class TrustBadgeShown(Scorer):)-class ProposedBadgeShown(Scorer):- """Binary: when the agent runs the named proposed metric, does the answer badge it and link to it?+_NONCANONICAL_BADGES = {"proposed": PROPOSED_BADGE_ICON, "drifted": DRIFTED_BADGE_CODE_POINT}- Using a proposed metric is the agent's call, so a case where it never runs the metric is skipped- (``score=None``) and left to the behavior judge.++class NoncanonicalBadgeShown(Scorer):+ """Binary: when the agent runs the named proposed or drifted metric, does the answer badge it and link to it?++ ``expected["proposed_badge"]["badge"]`` picks the badge (``proposed`` by default, or ``drifted``). Using+ such a metric is the agent's call, so a case where it never runs the metric is skipped (``score=None``)+ and left to the behavior judge."""def _name(self) -> str:+ # Both badges report under one name because PostHog upload accepts only registered scorer versions.return "proposed_badge"def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:@@ -1006,6 +1014,9 @@ class ProposedBadgeShown(Scorer):metric_name = spec.get("metric_name") if isinstance(spec, dict) else Noneif not isinstance(metric_name, str) or not metric_name:return Score(name=self._name(), score=0.0, metadata={"reason": "metric_name is required"})+ badge = _NONCANONICAL_BADGES.get(spec.get("badge", "proposed"))+ if badge is None:+ return Score(name=self._name(), score=0.0, metadata={"reason": "unknown badge"})parser = _parser_for(output)if parser is None:return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})@@ -1014,14 +1025,14 @@ class ProposedBadgeShown(Scorer):for call in parser.get_tool_calls(METRIC_RUN_TOOL))if not ran:- return Score(name=self._name(), score=None, metadata={"reason": "proposed metric not used"})+ return Score(name=self._name(), score=None, metadata={"reason": "metric not used"})answer = (output or {}).get("last_message") or ""opening = _first_line(answer)# A markdown link whose target ends at this metric's page, so a bare path or a metric whose# name only starts the same way does not count.link = re.compile(rf"\]\([^)\s]*/data-catalog/metrics/{re.escape(metric_name)}(?:[?#][^)\s]*)?\)")- has_badge = PROPOSED_BADGE_ICON in opening+ has_badge = badge in openinghas_link = bool(link.search(opening))return Score(name=self._name(),diff --git a/products/data_catalog/evals/seeders.py b/products/data_catalog/evals/seeders.pyindex 295bb4a3c548..35675a5d933c 100644--- a/products/data_catalog/evals/seeders.py+++ b/products/data_catalog/evals/seeders.py@@ -68,6 +68,8 @@ from products.data_catalog.evals.constants import (DRIFTED_INSIGHT_ORIGINAL_QUERY,DRIFTED_METRIC_DESCRIPTION,DRIFTED_METRIC_NAME,+ DRIFTED_MISMATCHED_INSIGHT_MUTATED_QUERY,+ DRIFTED_MISMATCHED_INSIGHT_ORIGINAL_QUERY,FAILING_TOP_CUSTOMERS_METRIC_DEFINITION,FEEDBACK_BY_SURVEY_METRIC_DEFINITION,FEEDBACK_BY_SURVEY_METRIC_DESCRIPTION,@@ -102,6 +104,10 @@ from products.data_catalog.evals.constants import (PROPOSED_METRIC_DEFINITION,PROPOSED_METRIC_DESCRIPTION,PROPOSED_METRIC_NAME,+ PROPOSED_UPLOADERS_METRIC_DEFINITION,+ PROPOSED_UPLOADERS_METRIC_DESCRIPTION,+ PROPOSED_UPLOADERS_METRIC_NAME,+ PROPOSED_UPLOADERS_MISMATCHED_DEFINITION,RELATIONSHIP_DECOY_TARGET_NAME,RELATIONSHIP_SOURCE_KEY,RELATIONSHIP_SOURCE_NAME,@@ -177,8 +183,10 @@ __all__ = ["seed_instruction_like_relationship_context","seed_metric_listing_catalog","seed_mcp_tool_call_fail_pct_metric",+ "seed_mismatched_drifted_metric",+ "seed_mismatched_proposed_uploaders_metric","seed_operational_metric",- "seed_proposed_metric",+ "seed_proposed_uploaders_metric","seed_subscription_snapshot_tables","seed_synced_invoices_table","seed_synced_time_off_table",@@ -511,22 +519,38 @@ def seed_proposed_growth_with_approved_mrr(context: CustomPromptSandboxContext)}-def seed_proposed_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:+def _seed_proposed_uploaders_metric(context: CustomPromptSandboxContext, definition: dict) -> dict[str, Any]:team, user = _team_and_user(context)upsert_metric(team=team,user=user,- name=PROPOSED_METRIC_NAME,- description=PROPOSED_METRIC_DESCRIPTION,- definition=PROPOSED_METRIC_DEFINITION,+ name=PROPOSED_UPLOADERS_METRIC_NAME,+ description=PROPOSED_UPLOADERS_METRIC_DESCRIPTION,+ definition=definition,overwrite=True,)- return {"metric": {"name": PROPOSED_METRIC_NAME, "status": "proposed"}}+ return {+ "metric": {+ "name": PROPOSED_UPLOADERS_METRIC_NAME,+ "status": "proposed",+ "definition_query": definition["query"],+ }+ }-def seed_drifted_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:+def seed_proposed_uploaders_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:+ return _seed_proposed_uploaders_metric(context, PROPOSED_UPLOADERS_METRIC_DEFINITION)+++def seed_mismatched_proposed_uploaders_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:+ return _seed_proposed_uploaders_metric(context, PROPOSED_UPLOADERS_MISMATCHED_DEFINITION)+++def _seed_drifted_metric(+ context: CustomPromptSandboxContext, original_query: dict, mutated_query: dict+) -> dict[str, Any]:team, user = _team_and_user(context)- insight = Insight.objects.create(team=team, created_by=user, query=DRIFTED_INSIGHT_ORIGINAL_QUERY)+ insight = Insight.objects.create(team=team, created_by=user, query=original_query)metric = upsert_metric(team=team,user=user,@@ -537,8 +561,25 @@ def seed_drifted_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:)approve_metric(metric, user)# Mutating the source insight after approval is what makes the metric read as drifted.- Insight.objects.filter(pk=insight.pk).update(query=DRIFTED_INSIGHT_MUTATED_QUERY)- return {"metric": {"name": DRIFTED_METRIC_NAME, "status": "approved", "is_drifted": True}}+ Insight.objects.filter(pk=insight.pk).update(query=mutated_query)+ return {+ "metric": {+ "name": DRIFTED_METRIC_NAME,+ "status": "approved",+ "is_drifted": True,+ "definition_query": original_query["query"],+ }+ }+++def seed_drifted_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:+ return _seed_drifted_metric(context, DRIFTED_INSIGHT_ORIGINAL_QUERY, DRIFTED_INSIGHT_MUTATED_QUERY)+++def seed_mismatched_drifted_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:+ return _seed_drifted_metric(+ context, DRIFTED_MISMATCHED_INSIGHT_ORIGINAL_QUERY, DRIFTED_MISMATCHED_INSIGHT_MUTATED_QUERY+ )def seed_definition_insight(context: CustomPromptSandboxContext) -> dict[str, Any]:diff --git a/products/posthog_ai/eval_harness/test/test_data_catalog_scorers.py b/products/posthog_ai/eval_harness/test/test_data_catalog_scorers.pyindex 900c0bb910bc..e77d7a921176 100644--- a/products/posthog_ai/eval_harness/test/test_data_catalog_scorers.py+++ b/products/posthog_ai/eval_harness/test/test_data_catalog_scorers.py@@ -22,7 +22,7 @@ from products.data_catalog.evals.scorers import (MetricsCatalogBeforeAnswer,MetricsCatalogBeforeDataDiscovery,MetricsCatalogQueried,- ProposedBadgeShown,+ NoncanonicalBadgeShown,SemanticMetadataQueried,TrustBadgeShown,)@@ -552,7 +552,7 @@ def test_answer_table_queried(_name: str, queries: list[tuple[str, str]], expect(AnswerTableQueried(), "answer_table_queried"),(MetricDescriptionConcise(), "metric_description_concise"),(TrustBadgeShown(), "trust_badge"),- (ProposedBadgeShown(), "proposed_badge"),+ (NoncanonicalBadgeShown(), "proposed_badge"),])def test_new_catalog_scorers_self_skip_when_not_requested(scorer: Any, scorer_name: str) -> None:@@ -626,13 +626,33 @@ _PROPOSED_LINK = "[Activation](https://example.com/project/1/data-catalog/metricdef test_proposed_badge_shown(_name: str, run_status: str, answer: str, expected_score: float | None) -> None:log = _tool_log([("data-catalog-metric-run", {"name": "activation_rate"}, run_status)])- score = ProposedBadgeShown()._run_eval_sync(+ score = NoncanonicalBadgeShown()._run_eval_sync({"raw_log": log, "last_message": answer}, {"proposed_badge": {"metric_name": "activation_rate"}})assert score.score == expected_score+_DRIFTED_LINK = "[Weekly active users](https://example.com/project/1/data-catalog/metrics/weekly_active_users)"+++ [+ ("warning_badge", f"\u26a0\ufe0f **Drifted definition in your data catalog**: {_DRIFTED_LINK}", 1.0),+ ("proposed_badge_instead", f"\U0001f4dd **Proposed definition in your data catalog**: {_DRIFTED_LINK}", 0.0),+ ]+)+def test_drifted_badge_shown(_name: str, answer: str, expected_score: float) -> None:+ log = _tool_log([("data-catalog-metric-run", {"name": "weekly_active_users"}, "completed")])++ score = NoncanonicalBadgeShown()._run_eval_sync(+ {"raw_log": log, "last_message": answer},+ {"proposed_badge": {"metric_name": "weekly_active_users", "badge": "drifted"}},+ )++ assert score.score == expected_score++_SHORT_DESCRIPTION = "Canonical MRR over the trailing 30 days, excluding personal/free plans."_LONG_DESCRIPTION = "x" * (EVAL_DESCRIPTION_CHAR_LIMIT + 1)