:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(metrics): check generated dashboard variables and panel runs (#120434)
Frank Hamandc35b2355663d
diff --git a/products/metrics/backend/suggested_dashboards/checks.py b/products/metrics/backend/suggested_dashboards/checks.pynew file mode 100644index 000000000000..674b233083f9--- /dev/null+++ b/products/metrics/backend/suggested_dashboards/checks.py@@ -0,0 +1,288 @@+"""Checks of a generated dashboard against the data of its source project.++The attribute profile finds the attributes that suit a dashboard variable, and the attributes with one value,+which make a useless variable. A pair check finds two variables that select the same thing. A panel run finds+the panels whose query fails as a dashboard tile runs it.+"""++from __future__ import annotations++import time+import datetime as dt+import contextvars+from collections.abc import Mapping, Sequence+from concurrent.futures import ThreadPoolExecutor, wait+from typing import Any++from django.db import connections++import structlog++from posthog.schema import DashboardFilter, DateRange, MetricsDashboardVariable, MetricsQuery++from posthog.hogql import ast+from posthog.hogql.constants import HogQLGlobalSettings+from posthog.hogql.database.schema.metrics import HOGQL_MAX_BYTES_TO_READ_FOR_METRICS_USER_QUERIES+from posthog.hogql.errors import ExposedHogQLError+from posthog.hogql.parser import parse_expr, parse_select+from posthog.hogql.query import execute_hogql_query++from posthog.clickhouse.client.connection import Workload+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded+from posthog.dataclasses import frozen+from posthog.errors import CH_TRANSIENT_ERRORS+from posthog.models import Team++from products.metrics.backend.hogql_queries.metrics_query_runner import MetricsQueryRunner+from products.metrics.backend.query_concurrency import query_concurrency_slots++logger = structlog.get_logger(__name__)++PROFILE_LOOKBACK = dt.timedelta(hours=24)+RUN_DATE_FROM = "-1h"+RUN_WORKERS = 4+# A panel run that has not started by then is left unchecked, so that the checks end in a bounded time.+RUN_BUDGET_SECONDS = 180.0+MIN_VARIABLE_VALUES = 2+MAX_CANDIDATE_VALUES = 20+MAX_CANDIDATES = 8+# Without a LIMIT, HogQL returns 100 rows. A key past this many is unknown, not a key with no values.+MAX_PROFILE_KEYS = 10_000+# Two attributes are one to one when they form at most 10% more pairs than the attribute with fewer values has+# values, and at most 10% of the series carry only one of them.+ONE_TO_ONE_TOLERANCE = 0.1+SERVICE_ATTRIBUTE = "service_name"+_SERVICE_ATTRIBUTES = frozenset({"service_name", "service.name"})+_QUERY_SETTINGS = HogQLGlobalSettings(max_bytes_to_read=HOGQL_MAX_BYTES_TO_READ_FOR_METRICS_USER_QUERIES)+++@frozen+class AttributeStats:+ value_count: int+ metric_count: int+++def _attribute(name: str) -> str:+ return SERVICE_ATTRIBUTE if name in _SERVICE_ATTRIBUTES else name+++def _names(metric_names: Sequence[str]) -> ast.Tuple:+ return ast.Tuple(exprs=[ast.Constant(value=name) for name in sorted(set(metric_names))])+++def _value(attribute: str) -> ast.Expr:+ if _attribute(attribute) == SERVICE_ATTRIBUTE:+ return ast.Field(chain=["service_name"])+ return parse_expr(+ "if(arrayElement(resource_attributes, {key}) != '', arrayElement(resource_attributes, {key}), "+ "arrayElement(attributes, {key}))",+ placeholders={"key": ast.Constant(value=attribute)},+ )+++def _since() -> dt.datetime:+ return dt.datetime.now(dt.UTC) - PROFILE_LOOKBACK+++def attribute_profile(team: Team, metric_names: Sequence[str]) -> dict[str, AttributeStats]:+ """The number of values of each attribute of the metrics, and the number of the metrics that have it."""+ query = parse_select(+ """+ SELECT attribute_key, uniqCombined64(attribute_value), uniqCombined64(metric_name)+ FROM (+ SELECT+ metric_name,+ arrayJoin(arrayConcat(+ mapKeys(attributes),+ arrayFilter(name -> name != 'service.name', mapKeys(resource_attributes)),+ ['service_name']+ )) AS attribute_key,+ if(attribute_key = 'service_name', service_name,+ if(arrayElement(resource_attributes, attribute_key) != '',+ arrayElement(resource_attributes, attribute_key),+ arrayElement(attributes, attribute_key))) AS attribute_value+ FROM posthog.metric_series+ WHERE last_seen >= {since} AND metric_name IN {names}+ )+ WHERE attribute_value != ''+ GROUP BY attribute_key+ LIMIT {limit}+ """,+ placeholders={+ "since": ast.Constant(value=_since()),+ "names": _names(metric_names),+ "limit": ast.Constant(value=MAX_PROFILE_KEYS),+ },+ )+ response = execute_hogql_query(+ query_type="MetricsDashboardAttributeProfile",+ query=query,+ team=team,+ workload=Workload.LOGS,+ settings=_QUERY_SETTINGS,+ )+ return {+ str(key): AttributeStats(value_count=int(values), metric_count=int(metrics))+ for key, values, metrics in response.results+ }+++def variable_candidates(profile: Mapping[str, AttributeStats], metric_count: int) -> list[str]:+ """The attributes that most of the metrics have, with a few values each: the ones a person would pick from."""+ candidates = [+ (key, stats)+ for key, stats in profile.items()+ if MIN_VARIABLE_VALUES <= stats.value_count <= MAX_CANDIDATE_VALUES and stats.metric_count * 2 > metric_count+ ]+ candidates.sort(key=lambda item: (-item[1].metric_count, item[1].value_count, item[0]))+ return [key for key, _ in candidates[:MAX_CANDIDATES]]+++def one_to_one(team: Team, metric_names: Sequence[str], first: str, second: str) -> bool:+ """Whether each value of one attribute goes with one value of the other, in nearly every series.++ A series with only one of the two attributes is one that only that attribute selects.+ """+ query = parse_select(+ """+ SELECT+ uniqIf(a, paired), uniqIf(b, paired), uniqIf(a, b, paired),+ countIf(paired), countIf(not paired)+ FROM (+ SELECT {first} AS a, {second} AS b, a != '' AND b != '' AS paired+ FROM posthog.metric_series+ WHERE last_seen >= {since} AND metric_name IN {names}+ )+ WHERE a != '' OR b != ''+ """,+ placeholders={+ "first": _value(first),+ "second": _value(second),+ "since": ast.Constant(value=_since()),+ "names": _names(metric_names),+ },+ )+ response = execute_hogql_query(+ query_type="MetricsDashboardVariablePair",+ query=query,+ team=team,+ workload=Workload.LOGS,+ settings=_QUERY_SETTINGS,+ )+ first_count, second_count, pairs, together, apart = (int(value) for value in response.results[0])+ fewer = min(first_count, second_count)+ return (+ fewer > 0 and pairs <= fewer * (1 + ONE_TO_ONE_TOLERANCE) and apart <= (together + apart) * ONE_TO_ONE_TOLERANCE+ )+++def _run(+ team: Team,+ query: dict[str, Any],+ variables: Sequence[MetricsDashboardVariable],+ *,+ timeout: float,+ start_by: float,+) -> str | None:+ """The error of the panel query, or None when it runs or was not checked.++ A transient ClickHouse failure says nothing about the panel, so it propagates and the activity retries.+ """+ started = time.monotonic()+ if started > start_by:+ return None+ try:+ runner = MetricsQueryRunner(+ query=MetricsQuery.model_validate({**query, "dateRange": DateRange(date_from=RUN_DATE_FROM)}),+ team=team,+ )+ if variables:+ runner.apply_dashboard_filters(DashboardFilter(metricVariables=list(variables)))+ with query_concurrency_slots(team):+ runner.calculate()+ except ConcurrencyLimitExceeded:+ # A busy organization says nothing about the panel, so it stays.+ logger.warning("metrics_suggested_dashboards_panel_run_skipped", team_id=team.id)+ return None+ except ExposedHogQLError as error:+ return str(error)+ except CH_TRANSIENT_ERRORS:+ raise+ except Exception:+ logger.exception("metrics_suggested_dashboards_panel_run_failed", team_id=team.id)+ return "The query could not run."+ finally:+ connections.close_all()+ if time.monotonic() - started > timeout:+ return "The query took too long to run."+ return None+++def _run_in(+ context: contextvars.Context,+ team: Team,+ query: dict[str, Any],+ variables: Sequence[MetricsDashboardVariable],+ *,+ timeout: float,+ start_by: float,+) -> str | None:+ return context.run(lambda: _run(team, query, variables, timeout=timeout, start_by=start_by))+++class ProjectChecks:+ """The checks for one generation. The profile comes from `attribute_profile`, or is None when it failed."""++ def __init__(+ self, team: Team, metric_names: Sequence[str], value_counts: Mapping[str, int] | None, *, deadline: float+ ) -> None:+ self._team = team+ self._metric_names = tuple(metric_names)+ self._value_counts = value_counts+ self._deadline = deadline++ def knows(self, attribute: str) -> bool:+ return self._value_counts is not None and _attribute(attribute) in self._value_counts++ def has_several_values(self, attribute: str) -> bool:+ count = self._value_counts.get(_attribute(attribute)) if self._value_counts is not None else None+ return count is None or count >= MIN_VARIABLE_VALUES++ def one_to_one(self, first: str, second: str) -> bool:+ if _attribute(first) == _attribute(second):+ return True+ try:+ return one_to_one(self._team, self._metric_names, first, second)+ except Exception:+ logger.warning("metrics_suggested_dashboards_variable_pair_failed", team_id=self._team.id)+ return False++ def run_errors(self, queries: Mapping[str, dict[str, Any]], variables: Sequence[dict[str, Any]]) -> dict[str, str]:+ """The error of each query that fails, with every variable set to All.++ A run counts as too slow when it takes longer than the deadline from its own start, so a panel that waits+ for a worker is not held against it. Every run finishes before this returns, so that no query of this+ generation runs on after it. ClickHouse ends each statement of a metrics query after 30 seconds.+ """+ definitions = [MetricsDashboardVariable.model_validate(variable) for variable in variables]+ start_by = time.monotonic() + RUN_BUDGET_SECONDS+ errors: dict[str, str] = {}+ with ThreadPoolExecutor(max_workers=RUN_WORKERS, thread_name_prefix="metrics-suggested-check") as executor:+ futures = {+ # Each run keeps the query tags of the activity, in its own copy of the context.+ key: executor.submit(+ _run_in,+ contextvars.copy_context(),+ self._team,+ query,+ definitions,+ timeout=self._deadline,+ start_by=start_by,+ )+ for key, query in queries.items()+ }+ wait(futures.values())+ for key, future in futures.items():+ if (error := future.result()) is not None:+ errors[key] = error+ return errorsdiff --git a/products/metrics/backend/suggested_dashboards/generation.py b/products/metrics/backend/suggested_dashboards/generation.pyindex 65788a891b88..3bdd6ca45c74 100644--- a/products/metrics/backend/suggested_dashboards/generation.py+++ b/products/metrics/backend/suggested_dashboards/generation.py@@ -40,6 +40,7 @@ from products.metrics.backend.metric_attributes_query_runner import ()from products.metrics.backend.models import MetricsDashboardTemplatefrom products.metrics.backend.suggested_dashboards import prompts+from products.metrics.backend.suggested_dashboards.checks import ProjectChecks, attribute_profile, variable_candidatesfrom products.metrics.backend.suggested_dashboards.dashboards import (SERVICE_ATTRIBUTES,acting_user,@@ -60,7 +61,13 @@ from products.metrics.backend.suggested_dashboards.spec import (template_panels,)from products.metrics.backend.suggested_dashboards.timeouts import GENERATE_TIMEOUT-from products.metrics.backend.variables import RANGE_VARIABLES, VARIABLE_NAME, referenced_names+from products.metrics.backend.variables import (+ RANGE_VARIABLES,+ REFERENCE,+ VARIABLE_NAME,+ reference_name,+ referenced_names,+)logger = structlog.get_logger(__name__)@@ -187,34 +194,83 @@ class _Converted:accepted_variables: tuple[DraftVariable, ...] = ()-def _valid_variables(drafts: Sequence[DraftVariable], attributes: Mapping[str, Sequence[str]]) -> list[DraftVariable]:- """The drafted variables that list a known attribute, with unique names."""+@frozen+class _ValidVariables:+ variables: list[DraftVariable]+ # Each variable that selects the same thing as a kept one, by name, with that kept variable.+ replaced: dict[str, tuple[DraftVariable, DraftVariable]]+++def _valid_variables(+ drafts: Sequence[DraftVariable], attributes: Mapping[str, Sequence[str]], checks: ProjectChecks | None+) -> _ValidVariables:+ """The drafted variables that list a known attribute with several values, with unique names.++ A variable whose values go one to one with those of an earlier variable selects the same thing, so the earlier+ one replaces it.+ """known = {key for keys in attributes.values() for key in keys} | SERVICE_ATTRIBUTESvalid: list[DraftVariable] = []+ replaced: dict[str, tuple[DraftVariable, DraftVariable]] = {}for draft in drafts:if len(valid) >= MAX_VARIABLES:breakif (not VARIABLE_NAME.match(draft.name)or draft.name in RANGE_VARIABLES- or draft.attribute not in known+ or (draft.attribute not in known and not (checks is not None and checks.knows(draft.attribute)))or any(variable.name == draft.name for variable in valid)):continue+ if checks is not None and not checks.has_several_values(draft.attribute):+ continue+ equivalent = (+ next((variable for variable in valid if checks.one_to_one(variable.attribute, draft.attribute)), None)+ if checks is not None+ else None+ )+ if equivalent is not None:+ replaced[draft.name] = (draft, equivalent)+ continuevalid.append(draft)- return valid+ return _ValidVariables(variables=valid, replaced=replaced)-def _with_known_references(draft: DraftPanel, names: set[str]) -> DraftPanel:- """The panel without the filters that refer to a variable that the dashboard does not have."""+def _same_attribute(first: str, second: str) -> bool:+ return first == second or {first, second} <= SERVICE_ATTRIBUTES- def known(builder_filter: BuilderFilter) -> bool:- return set(referenced_names(builder_filter.value)) <= names- clauses = [- clause.model_copy(update={"filters": [item for item in clause.filters if known(item)]})- for clause in draft.query.clauses- ]+def _replace_reference(builder_filter: BuilderFilter, old: DraftVariable, new: DraftVariable) -> BuilderFilter:+ value = REFERENCE.sub(+ lambda match: (+ match.group(0).replace(old.name, new.name, 1) if reference_name(match) == old.name else match.group(0)+ ),+ builder_filter.value,+ )+ key = new.attribute if _same_attribute(builder_filter.key, old.attribute) else builder_filter.key+ return builder_filter.model_copy(update={"key": key, "value": value})+++def _with_known_references(draft: DraftPanel, valid: _ValidVariables) -> DraftPanel:+ """The panel without the filters that refer to a variable that the dashboard does not have.++ A filter on a replaced variable filters on the variable that replaced it.+ """+ names = {variable.name for variable in valid.variables}++ def rewritten(builder_filter: BuilderFilter) -> BuilderFilter:+ for name in referenced_names(builder_filter.value):+ if name in valid.replaced:+ builder_filter = _replace_reference(builder_filter, *valid.replaced[name])+ return builder_filter++ clauses = []+ for clause in draft.query.clauses:+ filters: list[BuilderFilter] = []+ for item in map(rewritten, clause.filters):+ if set(referenced_names(item.value)) <= names and item not in filters:+ filters.append(item)+ clauses.append(clause.model_copy(update={"filters": filters}))return draft.model_copy(update={"query": draft.query.model_copy(update={"clauses": clauses})})@@ -225,22 +281,34 @@ def _convert(*,variables: Sequence[DraftVariable] = (),attributes: Mapping[str, Sequence[str]] | None = None,+ checks: ProjectChecks | None = None,) -> _Converted:"""Check each drafted query as PostHog would run it, and keep the panels that pass, with no overlapping boxes.A drafted variable stays when it lists a known attribute and a kept panel refers to it. It selects All, because- the template is for every project and a value of the source project must not reach the others.+ the template is for every project and a value of the source project must not reach the others. With `checks`,+ each panel also runs in the source project, and a panel whose run fails goes."""- valid_variables = _valid_variables(variables, attributes or {})- drafts = [_with_known_references(draft, {variable.name for variable in valid_variables}) for draft in drafts]+ valid = _valid_variables(variables, attributes or {}, checks)+ valid_variables = valid.variables+ definitions = {+ variable.name: attribute_variable(+ name=variable.name,+ label=" ".join(variable.label.split())[:100],+ attribute=variable.attribute,+ multi=variable.multi,+ )+ for variable in valid_variables+ }+ drafts = [_with_known_references(draft, valid) for draft in drafts]validator = PanelValidator(team=team, catalog=catalog, promql_available=False)queries = {draft.key: PanelQuery(language="builder", builder=draft.query) for draft in drafts}- checks = validator.check_all(queries, deadline_seconds=CHECK_DEADLINE_SECONDS)+ static_checks = validator.check_all(queries, deadline_seconds=CHECK_DEADLINE_SECONDS)accepted: list[DraftPanel] = []queries_by_key: dict[str, dict] = {}dropped: list[str] = []for draft in drafts:- check = checks.get(draft.key)+ check = static_checks.get(draft.key)if draft.key in queries_by_key or check is None or not check.ok:dropped.append(draft.title)continue@@ -252,6 +320,12 @@ def _convert(dropped.append(draft.title)continueaccepted.append(draft)+ if checks is not None and accepted:+ errors = checks.run_errors(+ {draft.key: queries_by_key[draft.key] for draft in accepted}, list(definitions.values())+ )+ dropped.extend(f"{draft.title}: {errors[draft.key]}" for draft in accepted if draft.key in errors)+ accepted = [draft for draft in accepted if draft.key not in errors]boxes = place_screenshot_boxes([RequestedBox(key=draft.key, layout=draft.layout, min_w=2, min_h=2) for draft in accepted])@@ -271,15 +345,7 @@ def _convert(panels=panels,accepted=tuple(accepted),dropped=tuple(dropped),- variables=tuple(- attribute_variable(- name=variable.name,- label=" ".join(variable.label.split())[:100],- attribute=variable.attribute,- multi=variable.multi,- )- for variable in kept_variables- ),+ variables=tuple(definitions[variable.name] for variable in kept_variables),accepted_variables=kept_variables,)@@ -292,11 +358,26 @@ def _store_panels(template: MetricsDashboardTemplate, record: GenerationRecord,record.dropped_panels = list(converted.dropped)+def _checks(context: _Context, value_counts: Mapping[str, int] | None) -> ProjectChecks:+ return ProjectChecks(+ context.team, [entry.name for entry in context.entries], value_counts, deadline=CHECK_DEADLINE_SECONDS+ )++def draft_template(template_id: str) -> int:"""Ask the model for the first version of the panels. Returns how many panels passed the checks."""context = _context(template_id)template, record = context.template, _record(context.template)attributes = _attributes(context.team, context.entries)+ metric_names = [entry.name for entry in context.entries]+ try:+ profile = attribute_profile(context.team, metric_names)+ except Exception:+ # Without the profile, the dashboard gets no candidates, and a variable is not checked for its values.+ logger.warning("metrics_suggested_dashboards_attribute_profile_failed", team_id=context.team.id)+ profile = None+ candidates = variable_candidates(profile or {}, len(metric_names))+ value_counts = {key: stats.value_count for key, stats in profile.items()} if profile is not None else Nonedraft = ask_model(team_id=context.team.id,stage="draft",@@ -309,6 +390,7 @@ def draft_template(template_id: str) -> int:entries=context.entries,attributes=attributes.keys,attribute_values=attributes.values,+ variable_candidates=candidates,))],@@ -320,12 +402,15 @@ def draft_template(template_id: str) -> int:context.catalog,variables=draft.variables,attributes=attributes.keys,+ checks=_checks(context, value_counts),)template.name = " ".join(draft.name.split())[:200] or template.nametemplate.description = " ".join(draft.description.split())[:300] or template.description_store_panels(template, record, converted)record.attributes = attributes.keysrecord.attribute_values = attributes.values+ record.attribute_value_counts = value_counts+ record.variable_candidates = candidates_save_record(template, record, "name", "description", "panels", "variables")return len(converted.panels)@@ -429,6 +514,7 @@ def check_preview(template_id: str, round_number: int, *, revise: bool) -> bool:entries=context.entries,attributes=record.attributes,attribute_values=record.attribute_values,+ variable_candidates=record.variable_candidates,dropped=record.dropped_panels,)),@@ -445,6 +531,7 @@ def check_preview(template_id: str, round_number: int, *, revise: bool) -> bool:context.catalog,variables=critique.variables,attributes=record.attributes,+ checks=_checks(context, record.attribute_value_counts),)if converted.panels:current.revised = Truediff --git a/products/metrics/backend/suggested_dashboards/prompts.py b/products/metrics/backend/suggested_dashboards/prompts.pyindex 65f6ac776996..27ec9c5c1254 100644--- a/products/metrics/backend/suggested_dashboards/prompts.py+++ b/products/metrics/backend/suggested_dashboards/prompts.py@@ -58,6 +58,9 @@ Variables:- Add a dashboard variable when the metrics share an attribute that names one of several instances of the system, \such as a job, an instance, a namespace or a cluster. A user then picks one instance for the whole dashboard. \service_name, the service that sends the metric, is always available for a variable.+- variable_candidates lists the attributes that most of the metrics have, with 2 to 20 values. Prefer them. An \+attribute with one value makes no variable.+- Two variables never select the same thing, such as a host and its instance id. Keep the one a person knows.- Add no variable for an attribute that splits one instance, such as a route, a status or a mode. Use group_by \for those. Use at most 2 variables.- Every panel whose metrics have the attribute refers to the variable with a filter: key is the attribute, op is \@@ -92,6 +95,9 @@ Look for problems that a person would notice:- The layout has holes or overlaps, a half-width panel alone in a row, or stats outside the top row.- Two panels show the same thing.- A variable that no panel refers to, or a panel that does not refer to a variable whose attribute its metrics have.+- A variable whose attribute has one value, or two variables that select the same thing.+- A panel in panels_that_failed_checks: its query failed, with the error after the title. Fix the query or leave \+the panel out.When there is no such problem, set looks_good to true. A small style preference is not a problem.When there are problems, set looks_good to false, list them, and return the complete corrected panel list and \@@ -148,12 +154,14 @@ def draft_input(entries: Sequence[CatalogEntry],attributes: dict[str, list[str]],attribute_values: dict[str, list[str]],+ variable_candidates: Sequence[str],) -> str:return json.dumps({"dashboard": {"name": name, "description": description},"metrics": [{**_entry(entry), "attributes": attributes.get(entry.name, [])} for entry in entries],"attribute_values": attribute_values,+ "variable_candidates": list(variable_candidates),},indent=1,)@@ -166,6 +174,7 @@ def critique_input(entries: Sequence[CatalogEntry],attributes: dict[str, list[str]],attribute_values: dict[str, list[str]],+ variable_candidates: Sequence[str],dropped: Sequence[str],) -> str:payload: dict[str, Any] = {@@ -173,6 +182,7 @@ def critique_input("variables": [variable.model_dump(mode="json") for variable in variables],"metric_catalog": [{**_entry(entry), "attributes": attributes.get(entry.name, [])} for entry in entries],"attribute_values": attribute_values,+ "variable_candidates": list(variable_candidates),}if dropped:payload["panels_that_failed_checks"] = list(dropped)diff --git a/products/metrics/backend/suggested_dashboards/spec.py b/products/metrics/backend/suggested_dashboards/spec.pyindex 3095e43c3e8e..869419e6b2ce 100644--- a/products/metrics/backend/suggested_dashboards/spec.py+++ b/products/metrics/backend/suggested_dashboards/spec.py@@ -135,5 +135,8 @@ class GenerationRecord(_Model):draft_variables: list[DraftVariable] = Field(default_factory=list)attributes: dict[str, list[str]] = Field(default_factory=dict)attribute_values: dict[str, list[str]] = Field(default_factory=dict)+ # None when the attribute profile failed, so that a variable is not dropped for a count nobody read.+ attribute_value_counts: dict[str, int] | None = None+ variable_candidates: list[str] = Field(default_factory=list)dropped_panels: list[str] = Field(default_factory=list)error: str | None = Nonediff --git a/products/metrics/backend/tests/test_suggested_dashboards.py b/products/metrics/backend/tests/test_suggested_dashboards.pyindex 3fb24fb1247f..ec96d77a912b 100644--- a/products/metrics/backend/tests/test_suggested_dashboards.py+++ b/products/metrics/backend/tests/test_suggested_dashboards.py@@ -20,6 +20,7 @@ from products.metrics.backend.dashboard_import.spec import (GridLayout,)from products.metrics.backend.suggested_dashboards.bank import curated_templates+from products.metrics.backend.suggested_dashboards.checks import AttributeStats, ProjectChecks, variable_candidatesfrom products.metrics.backend.suggested_dashboards.dashboards import (attribute_variable,build_tiles,@@ -66,6 +67,24 @@ def _panel(return TemplatePanel(key=key, title=key, query=_query(*names, kind=kind), layout=GridLayout(x=x, y=y, w=w, h=h))+class _Checks(ProjectChecks):+ def __init__(+ self,+ value_counts: dict[str, int] | None,+ pairs: frozenset[frozenset[str]] = frozenset(),+ errors: dict[str, str] | None = None,+ ) -> None:+ super().__init__(Team(id=1), [], value_counts, deadline=0)+ self._pairs = pairs+ self._errors = errors or {}++ def one_to_one(self, first: str, second: str) -> bool:+ return frozenset({first, second}) in self._pairs++ def run_errors(self, queries: Any, variables: Any) -> dict[str, str]:+ return {key: error for key, error in self._errors.items() if key in queries}++class TestSuggestedDashboards(SimpleTestCase):@parameterized.expand([(template.key,) for template in curated_templates()])def test_bank_template_queries_are_valid_and_fit_the_grid(self, key: str) -> None:@@ -284,6 +303,116 @@ class TestSuggestedDashboards(SimpleTestCase):MetricsQueryFilter.model_validate(item) for item in orders["clauses"][0]["filters"]) == ["instance"]+ @parameterized.expand(+ [+ ("one_value", ["instance"], {"instance": 1}, frozenset(), []),+ ("several_values", ["instance"], {"instance": 3}, frozenset(), ["instance"]),+ ("unread_profile", ["instance"], None, frozenset(), ["instance"]),+ (+ "one_to_one",+ ["instance", "host"],+ {"instance": 3, "host": 3},+ {frozenset({"instance", "host"})},+ ["instance"],+ ),+ ("independent", ["instance", "host"], {"instance": 3, "host": 3}, frozenset(), ["instance", "host"]),+ ("profiled_attribute_outside_the_keys", ["env"], {"env": 3}, frozenset(), ["env"]),+ ]+ )+ def test_convert_keeps_variables_with_several_values_that_select_different_things(+ self,+ _name: str,+ attributes: list[str],+ value_counts: dict[str, int] | None,+ pairs: frozenset[frozenset[str]],+ kept: list[str],+ ) -> None:+ converted = _convert(+ [+ self._draft(+ "orders",+ "orders_total",+ 0,+ filters=[BuilderFilter(key=attribute, op="eq", value=f"${attribute}") for attribute in attributes],+ )+ ],+ Team(id=1),+ CATALOG,+ variables=[DraftVariable(name=attribute, label=attribute, attribute=attribute) for attribute in attributes],+ attributes={"orders_total": ["instance", "host"]},+ checks=_Checks(value_counts, frozenset(pairs)),+ )++ assert [variable["name"] for variable in converted.variables] == kept+ orders = converted.panels[0].query+ assert orders is not None+ assert (+ filter_referenced_names(+ MetricsQueryFilter.model_validate(item) for item in orders["clauses"][0].get("filters", [])+ )+ == kept+ )++ def test_convert_filters_on_the_kept_variable_where_a_panel_filtered_on_a_replaced_one(self) -> None:+ converted = _convert(+ [+ self._draft(+ "orders", "orders_total", 0, filters=[BuilderFilter(key="instance", op="eq", value="$instance")]+ ),+ self._draft(+ "queue", "queue_depth", 4, filters=[BuilderFilter(key="host", op="regex", value="${host:regex}")]+ ),+ self._draft(+ "cache", "orders_total", 8, filters=[BuilderFilter(key="service.name", op="eq", value="$service")]+ ),+ ],+ Team(id=1),+ CATALOG,+ variables=[+ DraftVariable(name="instance", label="Instance", attribute="instance"),+ DraftVariable(name="host", label="Host", attribute="host"),+ DraftVariable(name="service", label="Service", attribute="service_name"),+ ],+ attributes={"orders_total": ["instance", "host"]},+ checks=_Checks(+ {"instance": 3, "host": 3, "service_name": 3},+ frozenset({frozenset({"instance", "host"}), frozenset({"instance", "service_name"})}),+ ),+ )++ assert [variable["name"] for variable in converted.variables] == ["instance"]+ queue, cache = (panel.query for panel in converted.panels[1:])+ assert queue is not None and cache is not None+ assert queue["clauses"][0]["filters"] == [{"key": "instance", "op": "regex", "value": "${instance:regex}"}]+ assert cache["clauses"][0]["filters"] == [{"key": "instance", "op": "eq", "value": "$instance"}]++ def test_convert_drops_the_panels_whose_query_fails_to_run(self) -> None:+ converted = _convert(+ [self._draft("orders", "orders_total", 0), self._draft("queue", "queue_depth", 4)],+ Team(id=1),+ CATALOG,+ checks=_Checks({}, errors={"queue": "Bad regex"}),+ )++ assert [panel.key for panel in converted.panels] == ["orders"]+ assert converted.dropped == ("queue: Bad regex",)++ @parameterized.expand(+ [+ ("one_value", 1, 3, False),+ ("few_values_on_most_metrics", 5, 3, True),+ ("on_half_of_the_metrics", 5, 2, False),+ ("too_many_values", 21, 4, False),+ ("at_the_limit", 20, 4, True),+ ]+ )+ def test_variable_candidates_have_a_few_values_on_most_metrics(+ self, _name: str, value_count: int, metric_count: int, candidate: bool+ ) -> None:+ profile = {"env": AttributeStats(value_count=value_count, metric_count=metric_count)}++ assert (variable_candidates(profile, 4) == ["env"]) is candidate+@staticmethoddef _draft(key: str, metric: str, x: int, filters: list[BuilderFilter] | None = None) -> DraftPanel:return DraftPanel(diff --git a/products/metrics/backend/tests/test_suggested_dashboards_checks.py b/products/metrics/backend/tests/test_suggested_dashboards_checks.pynew file mode 100644index 000000000000..8ac55e522cc5--- /dev/null+++ b/products/metrics/backend/tests/test_suggested_dashboards_checks.py@@ -0,0 +1,103 @@+import datetime as dt++import pytest+from posthog.test.base import APIBaseTest, ClickhouseTestMixin+from unittest.mock import patch++from clickhouse_driver.errors import NetworkError++from posthog.clickhouse.query_tagging import get_query_tag_value, tags_context++from products.metrics.backend.hogql_queries.metrics_query_runner import MetricsQueryRunner+from products.metrics.backend.suggested_dashboards.checks import (+ ProjectChecks,+ _run,+ attribute_profile,+ one_to_one,+ variable_candidates,+)+from products.metrics.backend.suggested_dashboards.dashboards import attribute_variable+from products.metrics.backend.tests._seeder import seed_metric, truncate_metrics_tables++METRICS = ["orders_total", "queue_depth", "cache_hits"]+++class TestSuggestedDashboardChecks(ClickhouseTestMixin, APIBaseTest):+ def setUp(self) -> None:+ super().setUp()+ truncate_metrics_tables()+ now = dt.datetime.now(dt.UTC) - dt.timedelta(minutes=5)+ for metric, labels in [+ ("orders_total", {"env": "prod", "host": "h1", "instance": "i1", "zone": "z1"}),+ ("orders_total", {"env": "prod", "host": "h2", "instance": "i2", "zone": "z2"}),+ ("orders_total", {"env": "dev", "host": "h3", "instance": "i3", "zone": "z3"}),+ ("queue_depth", {"env": "dev", "host": "h1", "instance": "i1", "zone": "z1"}),+ ("cache_hits", {"region": "eu", "zone": "z4"}),+ ]:+ seed_metric(team_id=self.team.id, metric_name=metric, points=[(now, 1.0)], labels=labels)++ def test_profile_counts_values_and_metrics_of_each_attribute(self) -> None:+ profile = attribute_profile(self.team, METRICS)++ assert {key: (stats.value_count, stats.metric_count) for key, stats in profile.items()} == {+ "env": (2, 2),+ "host": (3, 2),+ "instance": (3, 2),+ "region": (1, 1),+ "zone": (4, 3),+ "service_name": (1, 3),+ }+ assert variable_candidates(profile, len(METRICS)) == ["zone", "env", "host", "instance"]++ def test_one_to_one_finds_attributes_that_select_the_same_series(self) -> None:+ assert one_to_one(self.team, METRICS, "host", "instance")+ assert not one_to_one(self.team, METRICS, "env", "host")+ # Only zone selects the cache_hits series.+ assert not one_to_one(self.team, METRICS, "host", "zone")++ def test_run_errors_reports_only_the_queries_that_fail(self) -> None:+ def query(op: str, value: str) -> dict:+ return {+ "kind": "MetricsQuery",+ "clauses": [+ {+ "name": "a",+ "metricName": "queue_depth",+ "aggregation": "avg",+ "filters": [{"key": "host", "op": op, "value": value}],+ }+ ],+ }++ checks = ProjectChecks(self.team, METRICS, None, deadline=30)+ errors = checks.run_errors(+ {"fine": query("eq", "$host"), "broken": query("eq", "$missing")},+ [attribute_variable(name="host", label="Host", attribute="host")],+ )++ assert list(errors) == ["broken"]++ def test_a_transient_failure_propagates_and_a_late_run_leaves_the_panel_unchecked(self) -> None:+ query = {"kind": "MetricsQuery", "clauses": [{"name": "a", "metricName": "queue_depth", "aggregation": "avg"}]}+ checks = ProjectChecks(self.team, METRICS, None, deadline=30)++ with (+ patch.object(MetricsQueryRunner, "calculate", side_effect=NetworkError("connection reset")),+ pytest.raises(NetworkError),+ ):+ checks.run_errors({"a": query}, [])+ assert _run(self.team, {"kind": "MetricsQuery", "clauses": []}, [], timeout=30, start_by=0) is None++ def test_each_run_keeps_the_query_tags_of_the_caller(self) -> None:+ query = {"kind": "MetricsQuery", "clauses": [{"name": "a", "metricName": "queue_depth", "aggregation": "avg"}]}+ seen: list[object] = []++ with (+ tags_context(team_id=4242),+ patch.object(+ MetricsQueryRunner, "calculate", side_effect=lambda: seen.append(get_query_tag_value("team_id"))+ ),+ ):+ ProjectChecks(self.team, METRICS, None, deadline=30).run_errors({"a": query, "b": query}, [])++ assert seen == [4242, 4242]