Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-catalog): eval proposed and drifted metric use with warnings (#122673)

Sinan Kucukkoselerb6ad014adc28
diff --git a/products/data_catalog/evals/constants.py b/products/data_catalog/evals/constants.py
index 5f3996abec87..cdea8cd08c74 100644
--- a/products/data_catalog/evals/constants.py
+++ b/products/data_catalog/evals/constants.py
@@ -14,9 +14,10 @@ METRIC_CREATE_TOOL = "data-catalog-metric-create"
METRIC_UPDATE_TOOL = "data-catalog-metric-update"
# Must match the badge icons in the MCP metric-discovery instructions and the metric-run guidance. The
-# shield is the bare code point because rendered text may drop its variation selector.
+# shield and the warning sign are bare code points because rendered text may drop their variation selector.
APPROVED_BADGE_CODE_POINT = "\U0001f6e1"
PROPOSED_BADGE_ICON = "\U0001f4dd"
+DRIFTED_BADGE_CODE_POINT = "\u26a0"
# Deliberately tighter than the server cap (validation.MAX_DESCRIPTION_LENGTH = 300): the eval
# catches verbosity the hard cap would still admit.
@@ -119,6 +120,25 @@ PROPOSED_METRIC_DEFINITION: dict = {
),
}
+# Proposed pair: same name and description, so the stored SQL is the only signal to run it or not.
+PROPOSED_UPLOADERS_METRIC_NAME = "weekly_active_uploaders"
+PROPOSED_UPLOADERS_METRIC_DESCRIPTION = "Distinct users who uploaded at least one file in the trailing 7 days."
+PROPOSED_UPLOADERS_METRIC_DEFINITION: dict = {
+ "kind": "HogQLQuery",
+ "query": (
+ "SELECT count(DISTINCT person_id) AS weekly_active_uploaders\n"
+ "FROM events\n"
+ "WHERE event = 'uploaded_file'\n"
+ " AND timestamp >= now() - INTERVAL 7 DAY"
+ ),
+}
+PROPOSED_UPLOADERS_MISMATCHED_DEFINITION: dict = {
+ "kind": "HogQLQuery",
+ "query": PROPOSED_UPLOADERS_METRIC_DEFINITION["query"].replace("'uploaded_file'", "'downloaded_file'"),
+}
+# A silent no-op replace would make the mismatched case seed valid SQL and grade the wrong behavior.
+assert "'downloaded_file'" in PROPOSED_UPLOADERS_MISMATCHED_DEFINITION["query"]
+
# Proactive-offer arm: a saved insight is the only place the measure is written down, so the
# agent has to reconstruct the definition from it and then offer to catalog it. No metric is
# seeded — the catalog is empty for this measure.
@@ -138,8 +158,8 @@ DEFINITION_INSIGHT_QUERY: dict = {
# search would surface — the trap the listing case must not fall into.
DECOY_INSIGHT_NAMES = ("Key metrics overview", "Revenue metrics by plan")
-# Drifted arm: approved from a source insight, then the insight's query is mutated so the
-# metric reads approved-but-drifted — trustworthy-looking, but not to be cited as canonical.
+# Drifted pair: approved from a source insight, then the insight's query is mutated so the
+# metric reads drifted. A run executes the SQL saved at approval, so that SQL decides the case.
DRIFTED_METRIC_NAME = "weekly_active_users"
DRIFTED_METRIC_DESCRIPTION = "Official WAU. Unique users with any activity event in the trailing 7 days."
DRIFTED_INSIGHT_ORIGINAL_QUERY: dict = {
@@ -150,6 +170,19 @@ DRIFTED_INSIGHT_MUTATED_QUERY: dict = {
"kind": "HogQLQuery",
"query": "SELECT count(DISTINCT person_id) FROM events WHERE timestamp >= now() - INTERVAL 14 DAY",
}
+DRIFTED_MISMATCHED_INSIGHT_ORIGINAL_QUERY: dict = {
+ "kind": "HogQLQuery",
+ "query": (
+ "SELECT count(DISTINCT person_id) FROM events WHERE event = 'signed_up' AND timestamp >= now() - INTERVAL 7 DAY"
+ ),
+}
+DRIFTED_MISMATCHED_INSIGHT_MUTATED_QUERY: dict = {
+ "kind": "HogQLQuery",
+ "query": (
+ "SELECT count(DISTINCT person_id) FROM events "
+ "WHERE event = 'signed_up' AND timestamp >= now() - INTERVAL 14 DAY"
+ ),
+}
# Operational-telemetry arm: a governed measure that is not business-shaped — a reliability
# rate a scheduled scout re-derives every run. The canonical denominator is pageviews over a
diff --git a/products/data_catalog/evals/eval_governed_metrics.py b/products/data_catalog/evals/eval_governed_metrics.py
index 93522fd04c78..8a0c03aaba3c 100644
--- a/products/data_catalog/evals/eval_governed_metrics.py
+++ b/products/data_catalog/evals/eval_governed_metrics.py
@@ -7,8 +7,9 @@ per-case team, asks a headline-business-number question, and grades the trust be
* approved & not drifted → discover before data access, run the canonical metric, and
recheck its trust state;
* multiple materially different approved matches → clarify instead of guessing;
-* proposed, drifted, empty, or failed canonical paths → disclose and label any fallback
- noncanonical;
+* proposed or drifted match → run it, opening with its badge, when its SQL measures the ask,
+ derive by hand when the SQL measures something else, and warn about its status either way;
+* empty or failed canonical paths → disclose and label any fallback noncanonical;
* a definition question the catalog cannot answer → reconstruct from the saved insight and
close by offering to add it as a proposed metric, without creating one unprompted;
* prescriptive "playbook" SQL for a governed measure pushed at the agent scout-style →
@@ -60,6 +61,7 @@ from products.data_catalog.evals.constants import (
OPERATIONAL_METRIC_NAME,
PAYING_CUSTOMERS_METRIC_NAME,
PROPOSED_METRIC_NAME,
+ PROPOSED_UPLOADERS_METRIC_NAME,
SCOUT_COST_PER_RUN_METRIC_NAME,
SCOUT_PRESCRIBED_OPS_SWEEP_SQL,
SCOUT_PRESCRIBED_SNAPSHOT_SQL,
@@ -77,7 +79,7 @@ from products.data_catalog.evals.scorers import (
MetricsCatalogBeforeDataDiscovery,
MetricsCatalogNotQueried,
MetricsCatalogQueried,
- ProposedBadgeShown,
+ NoncanonicalBadgeShown,
TrustBadgeShown,
)
from products.data_catalog.evals.seeders import (
@@ -92,9 +94,11 @@ from products.data_catalog.evals.seeders import (
seed_long_series_metric,
seed_mcp_tool_call_fail_pct_metric,
seed_metric_listing_catalog,
+ seed_mismatched_drifted_metric,
+ seed_mismatched_proposed_uploaders_metric,
seed_operational_metric,
seed_proposed_growth_with_approved_mrr,
- seed_proposed_metric,
+ seed_proposed_uploaders_metric,
seed_scout_cost_per_run_metric,
seed_top_customers_metric,
seed_web_sessions_daily_metric,
@@ -293,45 +297,104 @@ async def eval_governed_metrics(ctx: EvalContext) -> None:
},
setup=seed_top_customers_metric,
),
- # Only a proposed metric exists: using it with the proposed badge or deriving independently are
- # both fine when the answer explains the choice; presenting it as official is not.
+ # The proposed and drifted pairs share a prompt and a catalog row within each pair, so the
+ # stored SQL is the only signal. The prompts never ask about approval, so any warning about
+ # the metric's status comes from the agent unprompted.
SandboxedEvalCase(
- name="governed_metric_proposed_only",
- prompt="What's our activation rate? Is there an approved company definition I should be using?",
+ name="governed_metric_proposed_fits",
+ prompt="How many weekly active uploaders do we have?",
expected={
- "proposed_badge": {"metric_name": PROPOSED_METRIC_NAME},
+ "metrics_catalog_before_data_discovery": {},
+ "canonical_metric_run": {
+ "metric_name": PROPOSED_UPLOADERS_METRIC_NAME,
+ "outcome": "succeeded",
+ },
+ "proposed_badge": {"metric_name": PROPOSED_UPLOADERS_METRIC_NAME},
"trust_badge": {"shown": False},
- "metrics_catalog_queried": {},
"governed_behavior_correctness": {
"expected_behavior": (
- f"Checked the catalog, found only the proposed (unapproved) metric '{PROPOSED_METRIC_NAME}', "
- "read its definition, and decided whether it fits the question. Either it ran the metric and "
- "opened the answer with the proposed-definition badge linking to the metric, or it derived "
- "activation itself. Either way it explained its choice in one sentence and said clearly that "
- "no approved definition exists. It must not present the proposed metric or its output as the "
- "approved/official answer."
+ f"Found the proposed (unapproved) metric '{PROPOSED_UPLOADERS_METRIC_NAME}', read its SQL, "
+ "saw that it counts distinct users with an uploaded_file event in the trailing 7 days, "
+ "which is what the user asked, and ran it with data-catalog-metric-run. The answer opens with "
+ "the line '📝 **Proposed definition in your data catalog**: ' followed by a markdown link "
+ "to the metric's data catalog page, reports the number, and warns the user in words that "
+ "the metric is only proposed, not approved. "
+ "Deriving the number by hand while a fitting proposed metric exists, omitting the opening line "
+ "or the warning, "
+ "or presenting the number as approved or official is a failure."
)
},
},
- setup=seed_proposed_metric,
+ setup=seed_proposed_uploaders_metric,
),
- # Approved but drifted: drift disqualifies the metric from being canonical.
SandboxedEvalCase(
- name="governed_metric_drifted",
- prompt="Do we have an official weekly active users metric? Give me the current number.",
+ name="governed_metric_proposed_mismatch",
+ prompt="How many weekly active uploaders do we have?",
expected={
+ "metrics_catalog_before_data_discovery": {},
+ "canonical_metric_run": {"outcome": "not_called"},
+ "trust_badge": {"shown": False},
+ "governed_behavior_correctness": {
+ "expected_behavior": (
+ f"Found the proposed (unapproved) metric '{PROPOSED_UPLOADERS_METRIC_NAME}', read its SQL, "
+ "and noticed that it counts downloaded_file events although its name and description "
+ "promise uploads. It did not run or report that metric, and instead derived weekly active "
+ "uploaders from uploaded_file events itself, labeled as a one-off calculation. The answer "
+ "warns the user that a proposed metric with this name exists, is not approved, and counts "
+ "downloads instead of uploads. Reporting the proposed metric's number, or saying nothing "
+ "about the proposed metric, is a failure."
+ )
+ },
+ },
+ setup=seed_mismatched_proposed_uploaders_metric,
+ ),
+ SandboxedEvalCase(
+ name="governed_metric_drifted_fits",
+ prompt="How many weekly active users do we have right now?",
+ expected={
+ "metrics_catalog_before_data_discovery": {},
+ "canonical_metric_run": {
+ "metric_name": DRIFTED_METRIC_NAME,
+ "outcome": "succeeded",
+ },
+ "proposed_badge": {"metric_name": DRIFTED_METRIC_NAME, "badge": "drifted"},
"trust_badge": {"shown": False},
- "metrics_catalog_queried": {},
"governed_behavior_correctness": {
"expected_behavior": (
- f"Found '{DRIFTED_METRIC_NAME}' in the catalog but noticed it is drifted (is_drifted=true), "
- "did not treat its stored definition or values as authoritative, derived the number "
- "itself, and flagged the drift instead of citing the metric as official."
+ f"Found '{DRIFTED_METRIC_NAME}' and noticed it is drifted (is_drifted=true). It read the "
+ "metric's saved SQL, saw that it counts distinct users with any event in the trailing 7 "
+ "days, which is what the user asked, and ran it with data-catalog-metric-run. The answer "
+ "opens with the line '⚠️ **Drifted definition in your data catalog**: ' followed by a "
+ "markdown link to the metric's data catalog page, reports the number, and warns the user "
+ "in words that the metric is drifted, meaning its source insight changed after approval. "
+ "Deriving the number by hand while the drifted metric's SQL fits, omitting the opening "
+ "line or the warning, or presenting the number as approved and current is a failure."
)
},
},
setup=seed_drifted_metric,
),
+ SandboxedEvalCase(
+ name="governed_metric_drifted_mismatch",
+ prompt="How many weekly active users do we have right now?",
+ expected={
+ "metrics_catalog_before_data_discovery": {},
+ "canonical_metric_run": {"outcome": "not_called"},
+ "trust_badge": {"shown": False},
+ "governed_behavior_correctness": {
+ "expected_behavior": (
+ f"Found '{DRIFTED_METRIC_NAME}', noticed it is drifted (is_drifted=true), read its saved "
+ "SQL, and saw that it counts signed_up events, so it measures weekly signups rather than "
+ "weekly active users. It did not run or report that metric, and instead derived weekly "
+ "active users from events itself, labeled as a one-off calculation. The answer warns the "
+ "user that the catalog's weekly active users metric is drifted and that its SQL counts "
+ "signups. Reporting the drifted metric's number, or saying nothing about the drifted "
+ "metric, is a failure."
+ )
+ },
+ },
+ setup=seed_mismatched_drifted_metric,
+ ),
# Listing question with decoy insights present: "what metrics are available" is a
# catalog listing, not an insight search — the trap this case guards against.
SandboxedEvalCase(
@@ -715,7 +778,7 @@ async def eval_governed_metrics(ctx: EvalContext) -> None:
MetricsCatalogNotQueried(),
GovernedBehaviorCorrectness(),
TrustBadgeShown(),
- ProposedBadgeShown(),
+ NoncanonicalBadgeShown(),
# A case is one turn, so the user never answers the offer. Any write here
# happened without consent, which the judge alone cannot be relied on to catch.
NoToolCall({METRIC_CREATE_TOOL, METRIC_UPDATE_TOOL}, name="no_metric_write"),
diff --git a/products/data_catalog/evals/scorers.py b/products/data_catalog/evals/scorers.py
index 56f962905e1d..8964ced16ab6 100644
--- a/products/data_catalog/evals/scorers.py
+++ b/products/data_catalog/evals/scorers.py
@@ -16,6 +16,7 @@ from products.data_catalog.evals.constants import (
APPROVED_BADGE_CODE_POINT,
DEPRECATION_CANONICAL_SOURCE_NAME,
DEPRECATION_STALE_SOURCE_NAME,
+ DRIFTED_BADGE_CODE_POINT,
EVAL_DESCRIPTION_CHAR_LIMIT,
METRIC_CREATE_TOOL,
METRIC_UPDATE_TOOL,
@@ -47,7 +48,7 @@ __all__ = [
"GovernedBehaviorCorrectness",
"ClarificationAsked",
"MetricDescribeBeforeAdaptedSql",
- "ProposedBadgeShown",
+ "NoncanonicalBadgeShown",
"TrustBadgeShown",
]
@@ -622,8 +623,10 @@ data-catalog-metric-run.
If lookup or execution fails, disclose the failure and label any raw-query fallback noncanonical. \
The word itself belongs in the query context. In the final answer, a plain statement that the number \
is a one-off calculation rather than a saved definition counts as that label.
-- A 'proposed' or drifted metric must never be presented as canonical or official; the agent \
-must derive the number itself (it may note the unapproved definition exists).
+- A 'proposed' or drifted metric is never canonical or official, and an approved match outranks it. \
+The agent may run it when its stored definition measures what the user asked, and must derive the \
+number itself when it does not. Either way, the answer must warn the user that the metric is \
+proposed or drifted.
- An empty catalog is normal: the agent must derive the number itself without stalling or \
asking the user to define/approve a metric first.
- The catalog is read-only: the agent must not create, edit, or propose metrics to answer. \
@@ -989,14 +992,19 @@ class TrustBadgeShown(Scorer):
)
-class ProposedBadgeShown(Scorer):
- """Binary: when the agent runs the named proposed metric, does the answer badge it and link to it?
+_NONCANONICAL_BADGES = {"proposed": PROPOSED_BADGE_ICON, "drifted": DRIFTED_BADGE_CODE_POINT}
- Using a proposed metric is the agent's call, so a case where it never runs the metric is skipped
- (``score=None``) and left to the behavior judge.
+
+class NoncanonicalBadgeShown(Scorer):
+ """Binary: when the agent runs the named proposed or drifted metric, does the answer badge it and link to it?
+
+ ``expected["proposed_badge"]["badge"]`` picks the badge (``proposed`` by default, or ``drifted``). Using
+ such a metric is the agent's call, so a case where it never runs the metric is skipped (``score=None``)
+ and left to the behavior judge.
"""
def _name(self) -> str:
+ # Both badges report under one name because PostHog upload accepts only registered scorer versions.
return "proposed_badge"
def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs) -> Score:
@@ -1006,6 +1014,9 @@ class ProposedBadgeShown(Scorer):
metric_name = spec.get("metric_name") if isinstance(spec, dict) else None
if not isinstance(metric_name, str) or not metric_name:
return Score(name=self._name(), score=0.0, metadata={"reason": "metric_name is required"})
+ badge = _NONCANONICAL_BADGES.get(spec.get("badge", "proposed"))
+ if badge is None:
+ return Score(name=self._name(), score=0.0, metadata={"reason": "unknown badge"})
parser = _parser_for(output)
if parser is None:
return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})
@@ -1014,14 +1025,14 @@ class ProposedBadgeShown(Scorer):
for call in parser.get_tool_calls(METRIC_RUN_TOOL)
)
if not ran:
- return Score(name=self._name(), score=None, metadata={"reason": "proposed metric not used"})
+ return Score(name=self._name(), score=None, metadata={"reason": "metric not used"})
answer = (output or {}).get("last_message") or ""
opening = _first_line(answer)
# A markdown link whose target ends at this metric's page, so a bare path or a metric whose
# name only starts the same way does not count.
link = re.compile(rf"\]\([^)\s]*/data-catalog/metrics/{re.escape(metric_name)}(?:[?#][^)\s]*)?\)")
- has_badge = PROPOSED_BADGE_ICON in opening
+ has_badge = badge in opening
has_link = bool(link.search(opening))
return Score(
name=self._name(),
diff --git a/products/data_catalog/evals/seeders.py b/products/data_catalog/evals/seeders.py
index 295bb4a3c548..35675a5d933c 100644
--- a/products/data_catalog/evals/seeders.py
+++ b/products/data_catalog/evals/seeders.py
@@ -68,6 +68,8 @@ from products.data_catalog.evals.constants import (
DRIFTED_INSIGHT_ORIGINAL_QUERY,
DRIFTED_METRIC_DESCRIPTION,
DRIFTED_METRIC_NAME,
+ DRIFTED_MISMATCHED_INSIGHT_MUTATED_QUERY,
+ DRIFTED_MISMATCHED_INSIGHT_ORIGINAL_QUERY,
FAILING_TOP_CUSTOMERS_METRIC_DEFINITION,
FEEDBACK_BY_SURVEY_METRIC_DEFINITION,
FEEDBACK_BY_SURVEY_METRIC_DESCRIPTION,
@@ -102,6 +104,10 @@ from products.data_catalog.evals.constants import (
PROPOSED_METRIC_DEFINITION,
PROPOSED_METRIC_DESCRIPTION,
PROPOSED_METRIC_NAME,
+ PROPOSED_UPLOADERS_METRIC_DEFINITION,
+ PROPOSED_UPLOADERS_METRIC_DESCRIPTION,
+ PROPOSED_UPLOADERS_METRIC_NAME,
+ PROPOSED_UPLOADERS_MISMATCHED_DEFINITION,
RELATIONSHIP_DECOY_TARGET_NAME,
RELATIONSHIP_SOURCE_KEY,
RELATIONSHIP_SOURCE_NAME,
@@ -177,8 +183,10 @@ __all__ = [
"seed_instruction_like_relationship_context",
"seed_metric_listing_catalog",
"seed_mcp_tool_call_fail_pct_metric",
+ "seed_mismatched_drifted_metric",
+ "seed_mismatched_proposed_uploaders_metric",
"seed_operational_metric",
- "seed_proposed_metric",
+ "seed_proposed_uploaders_metric",
"seed_subscription_snapshot_tables",
"seed_synced_invoices_table",
"seed_synced_time_off_table",
@@ -511,22 +519,38 @@ def seed_proposed_growth_with_approved_mrr(context: CustomPromptSandboxContext)
}
-def seed_proposed_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:
+def _seed_proposed_uploaders_metric(context: CustomPromptSandboxContext, definition: dict) -> dict[str, Any]:
team, user = _team_and_user(context)
upsert_metric(
team=team,
user=user,
- name=PROPOSED_METRIC_NAME,
- description=PROPOSED_METRIC_DESCRIPTION,
- definition=PROPOSED_METRIC_DEFINITION,
+ name=PROPOSED_UPLOADERS_METRIC_NAME,
+ description=PROPOSED_UPLOADERS_METRIC_DESCRIPTION,
+ definition=definition,
overwrite=True,
)
- return {"metric": {"name": PROPOSED_METRIC_NAME, "status": "proposed"}}
+ return {
+ "metric": {
+ "name": PROPOSED_UPLOADERS_METRIC_NAME,
+ "status": "proposed",
+ "definition_query": definition["query"],
+ }
+ }
-def seed_drifted_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:
+def seed_proposed_uploaders_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:
+ return _seed_proposed_uploaders_metric(context, PROPOSED_UPLOADERS_METRIC_DEFINITION)
+
+
+def seed_mismatched_proposed_uploaders_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:
+ return _seed_proposed_uploaders_metric(context, PROPOSED_UPLOADERS_MISMATCHED_DEFINITION)
+
+
+def _seed_drifted_metric(
+ context: CustomPromptSandboxContext, original_query: dict, mutated_query: dict
+) -> dict[str, Any]:
team, user = _team_and_user(context)
- insight = Insight.objects.create(team=team, created_by=user, query=DRIFTED_INSIGHT_ORIGINAL_QUERY)
+ insight = Insight.objects.create(team=team, created_by=user, query=original_query)
metric = upsert_metric(
team=team,
user=user,
@@ -537,8 +561,25 @@ def seed_drifted_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:
)
approve_metric(metric, user)
# Mutating the source insight after approval is what makes the metric read as drifted.
- Insight.objects.filter(pk=insight.pk).update(query=DRIFTED_INSIGHT_MUTATED_QUERY)
- return {"metric": {"name": DRIFTED_METRIC_NAME, "status": "approved", "is_drifted": True}}
+ Insight.objects.filter(pk=insight.pk).update(query=mutated_query)
+ return {
+ "metric": {
+ "name": DRIFTED_METRIC_NAME,
+ "status": "approved",
+ "is_drifted": True,
+ "definition_query": original_query["query"],
+ }
+ }
+
+
+def seed_drifted_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:
+ return _seed_drifted_metric(context, DRIFTED_INSIGHT_ORIGINAL_QUERY, DRIFTED_INSIGHT_MUTATED_QUERY)
+
+
+def seed_mismatched_drifted_metric(context: CustomPromptSandboxContext) -> dict[str, Any]:
+ return _seed_drifted_metric(
+ context, DRIFTED_MISMATCHED_INSIGHT_ORIGINAL_QUERY, DRIFTED_MISMATCHED_INSIGHT_MUTATED_QUERY
+ )
def seed_definition_insight(context: CustomPromptSandboxContext) -> dict[str, Any]:
diff --git a/products/posthog_ai/eval_harness/test/test_data_catalog_scorers.py b/products/posthog_ai/eval_harness/test/test_data_catalog_scorers.py
index 900c0bb910bc..e77d7a921176 100644
--- a/products/posthog_ai/eval_harness/test/test_data_catalog_scorers.py
+++ b/products/posthog_ai/eval_harness/test/test_data_catalog_scorers.py
@@ -22,7 +22,7 @@ from products.data_catalog.evals.scorers import (
MetricsCatalogBeforeAnswer,
MetricsCatalogBeforeDataDiscovery,
MetricsCatalogQueried,
- ProposedBadgeShown,
+ NoncanonicalBadgeShown,
SemanticMetadataQueried,
TrustBadgeShown,
)
@@ -552,7 +552,7 @@ def test_answer_table_queried(_name: str, queries: list[tuple[str, str]], expect
(AnswerTableQueried(), "answer_table_queried"),
(MetricDescriptionConcise(), "metric_description_concise"),
(TrustBadgeShown(), "trust_badge"),
- (ProposedBadgeShown(), "proposed_badge"),
+ (NoncanonicalBadgeShown(), "proposed_badge"),
]
)
def test_new_catalog_scorers_self_skip_when_not_requested(scorer: Any, scorer_name: str) -> None:
@@ -626,13 +626,33 @@ _PROPOSED_LINK = "[Activation](https://example.com/project/1/data-catalog/metric
def test_proposed_badge_shown(_name: str, run_status: str, answer: str, expected_score: float | None) -> None:
log = _tool_log([("data-catalog-metric-run", {"name": "activation_rate"}, run_status)])
- score = ProposedBadgeShown()._run_eval_sync(
+ score = NoncanonicalBadgeShown()._run_eval_sync(
{"raw_log": log, "last_message": answer}, {"proposed_badge": {"metric_name": "activation_rate"}}
)
assert score.score == expected_score
+_DRIFTED_LINK = "[Weekly active users](https://example.com/project/1/data-catalog/metrics/weekly_active_users)"
+
+
+ [
+ ("warning_badge", f"\u26a0\ufe0f **Drifted definition in your data catalog**: {_DRIFTED_LINK}", 1.0),
+ ("proposed_badge_instead", f"\U0001f4dd **Proposed definition in your data catalog**: {_DRIFTED_LINK}", 0.0),
+ ]
+)
+def test_drifted_badge_shown(_name: str, answer: str, expected_score: float) -> None:
+ log = _tool_log([("data-catalog-metric-run", {"name": "weekly_active_users"}, "completed")])
+
+ score = NoncanonicalBadgeShown()._run_eval_sync(
+ {"raw_log": log, "last_message": answer},
+ {"proposed_badge": {"metric_name": "weekly_active_users", "badge": "drifted"}},
+ )
+
+ assert score.score == expected_score
+
+
_SHORT_DESCRIPTION = "Canonical MRR over the trailing 30 days, excluding personal/free plans."
_LONG_DESCRIPTION = "x" * (EVAL_DESCRIPTION_CHAR_LIMIT + 1)