Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(metrics): check generated dashboard variables and panel runs (#120434)

Frank Hamandc35b2355663d
diff --git a/products/metrics/backend/suggested_dashboards/checks.py b/products/metrics/backend/suggested_dashboards/checks.py
new file mode 100644
index 000000000000..674b233083f9
--- /dev/null
+++ b/products/metrics/backend/suggested_dashboards/checks.py
@@ -0,0 +1,288 @@
+"""Checks of a generated dashboard against the data of its source project.
+
+The attribute profile finds the attributes that suit a dashboard variable, and the attributes with one value,
+which make a useless variable. A pair check finds two variables that select the same thing. A panel run finds
+the panels whose query fails as a dashboard tile runs it.
+"""
+
+from __future__ import annotations
+
+import time
+import datetime as dt
+import contextvars
+from collections.abc import Mapping, Sequence
+from concurrent.futures import ThreadPoolExecutor, wait
+from typing import Any
+
+from django.db import connections
+
+import structlog
+
+from posthog.schema import DashboardFilter, DateRange, MetricsDashboardVariable, MetricsQuery
+
+from posthog.hogql import ast
+from posthog.hogql.constants import HogQLGlobalSettings
+from posthog.hogql.database.schema.metrics import HOGQL_MAX_BYTES_TO_READ_FOR_METRICS_USER_QUERIES
+from posthog.hogql.errors import ExposedHogQLError
+from posthog.hogql.parser import parse_expr, parse_select
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.clickhouse.client.connection import Workload
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
+from posthog.dataclasses import frozen
+from posthog.errors import CH_TRANSIENT_ERRORS
+from posthog.models import Team
+
+from products.metrics.backend.hogql_queries.metrics_query_runner import MetricsQueryRunner
+from products.metrics.backend.query_concurrency import query_concurrency_slots
+
+logger = structlog.get_logger(__name__)
+
+PROFILE_LOOKBACK = dt.timedelta(hours=24)
+RUN_DATE_FROM = "-1h"
+RUN_WORKERS = 4
+# A panel run that has not started by then is left unchecked, so that the checks end in a bounded time.
+RUN_BUDGET_SECONDS = 180.0
+MIN_VARIABLE_VALUES = 2
+MAX_CANDIDATE_VALUES = 20
+MAX_CANDIDATES = 8
+# Without a LIMIT, HogQL returns 100 rows. A key past this many is unknown, not a key with no values.
+MAX_PROFILE_KEYS = 10_000
+# Two attributes are one to one when they form at most 10% more pairs than the attribute with fewer values has
+# values, and at most 10% of the series carry only one of them.
+ONE_TO_ONE_TOLERANCE = 0.1
+SERVICE_ATTRIBUTE = "service_name"
+_SERVICE_ATTRIBUTES = frozenset({"service_name", "service.name"})
+_QUERY_SETTINGS = HogQLGlobalSettings(max_bytes_to_read=HOGQL_MAX_BYTES_TO_READ_FOR_METRICS_USER_QUERIES)
+
+
+@frozen
+class AttributeStats:
+ value_count: int
+ metric_count: int
+
+
+def _attribute(name: str) -> str:
+ return SERVICE_ATTRIBUTE if name in _SERVICE_ATTRIBUTES else name
+
+
+def _names(metric_names: Sequence[str]) -> ast.Tuple:
+ return ast.Tuple(exprs=[ast.Constant(value=name) for name in sorted(set(metric_names))])
+
+
+def _value(attribute: str) -> ast.Expr:
+ if _attribute(attribute) == SERVICE_ATTRIBUTE:
+ return ast.Field(chain=["service_name"])
+ return parse_expr(
+ "if(arrayElement(resource_attributes, {key}) != '', arrayElement(resource_attributes, {key}), "
+ "arrayElement(attributes, {key}))",
+ placeholders={"key": ast.Constant(value=attribute)},
+ )
+
+
+def _since() -> dt.datetime:
+ return dt.datetime.now(dt.UTC) - PROFILE_LOOKBACK
+
+
+def attribute_profile(team: Team, metric_names: Sequence[str]) -> dict[str, AttributeStats]:
+ """The number of values of each attribute of the metrics, and the number of the metrics that have it."""
+ query = parse_select(
+ """
+ SELECT attribute_key, uniqCombined64(attribute_value), uniqCombined64(metric_name)
+ FROM (
+ SELECT
+ metric_name,
+ arrayJoin(arrayConcat(
+ mapKeys(attributes),
+ arrayFilter(name -> name != 'service.name', mapKeys(resource_attributes)),
+ ['service_name']
+ )) AS attribute_key,
+ if(attribute_key = 'service_name', service_name,
+ if(arrayElement(resource_attributes, attribute_key) != '',
+ arrayElement(resource_attributes, attribute_key),
+ arrayElement(attributes, attribute_key))) AS attribute_value
+ FROM posthog.metric_series
+ WHERE last_seen >= {since} AND metric_name IN {names}
+ )
+ WHERE attribute_value != ''
+ GROUP BY attribute_key
+ LIMIT {limit}
+ """,
+ placeholders={
+ "since": ast.Constant(value=_since()),
+ "names": _names(metric_names),
+ "limit": ast.Constant(value=MAX_PROFILE_KEYS),
+ },
+ )
+ response = execute_hogql_query(
+ query_type="MetricsDashboardAttributeProfile",
+ query=query,
+ team=team,
+ workload=Workload.LOGS,
+ settings=_QUERY_SETTINGS,
+ )
+ return {
+ str(key): AttributeStats(value_count=int(values), metric_count=int(metrics))
+ for key, values, metrics in response.results
+ }
+
+
+def variable_candidates(profile: Mapping[str, AttributeStats], metric_count: int) -> list[str]:
+ """The attributes that most of the metrics have, with a few values each: the ones a person would pick from."""
+ candidates = [
+ (key, stats)
+ for key, stats in profile.items()
+ if MIN_VARIABLE_VALUES <= stats.value_count <= MAX_CANDIDATE_VALUES and stats.metric_count * 2 > metric_count
+ ]
+ candidates.sort(key=lambda item: (-item[1].metric_count, item[1].value_count, item[0]))
+ return [key for key, _ in candidates[:MAX_CANDIDATES]]
+
+
+def one_to_one(team: Team, metric_names: Sequence[str], first: str, second: str) -> bool:
+ """Whether each value of one attribute goes with one value of the other, in nearly every series.
+
+ A series with only one of the two attributes is one that only that attribute selects.
+ """
+ query = parse_select(
+ """
+ SELECT
+ uniqIf(a, paired), uniqIf(b, paired), uniqIf(a, b, paired),
+ countIf(paired), countIf(not paired)
+ FROM (
+ SELECT {first} AS a, {second} AS b, a != '' AND b != '' AS paired
+ FROM posthog.metric_series
+ WHERE last_seen >= {since} AND metric_name IN {names}
+ )
+ WHERE a != '' OR b != ''
+ """,
+ placeholders={
+ "first": _value(first),
+ "second": _value(second),
+ "since": ast.Constant(value=_since()),
+ "names": _names(metric_names),
+ },
+ )
+ response = execute_hogql_query(
+ query_type="MetricsDashboardVariablePair",
+ query=query,
+ team=team,
+ workload=Workload.LOGS,
+ settings=_QUERY_SETTINGS,
+ )
+ first_count, second_count, pairs, together, apart = (int(value) for value in response.results[0])
+ fewer = min(first_count, second_count)
+ return (
+ fewer > 0 and pairs <= fewer * (1 + ONE_TO_ONE_TOLERANCE) and apart <= (together + apart) * ONE_TO_ONE_TOLERANCE
+ )
+
+
+def _run(
+ team: Team,
+ query: dict[str, Any],
+ variables: Sequence[MetricsDashboardVariable],
+ *,
+ timeout: float,
+ start_by: float,
+) -> str | None:
+ """The error of the panel query, or None when it runs or was not checked.
+
+ A transient ClickHouse failure says nothing about the panel, so it propagates and the activity retries.
+ """
+ started = time.monotonic()
+ if started > start_by:
+ return None
+ try:
+ runner = MetricsQueryRunner(
+ query=MetricsQuery.model_validate({**query, "dateRange": DateRange(date_from=RUN_DATE_FROM)}),
+ team=team,
+ )
+ if variables:
+ runner.apply_dashboard_filters(DashboardFilter(metricVariables=list(variables)))
+ with query_concurrency_slots(team):
+ runner.calculate()
+ except ConcurrencyLimitExceeded:
+ # A busy organization says nothing about the panel, so it stays.
+ logger.warning("metrics_suggested_dashboards_panel_run_skipped", team_id=team.id)
+ return None
+ except ExposedHogQLError as error:
+ return str(error)
+ except CH_TRANSIENT_ERRORS:
+ raise
+ except Exception:
+ logger.exception("metrics_suggested_dashboards_panel_run_failed", team_id=team.id)
+ return "The query could not run."
+ finally:
+ connections.close_all()
+ if time.monotonic() - started > timeout:
+ return "The query took too long to run."
+ return None
+
+
+def _run_in(
+ context: contextvars.Context,
+ team: Team,
+ query: dict[str, Any],
+ variables: Sequence[MetricsDashboardVariable],
+ *,
+ timeout: float,
+ start_by: float,
+) -> str | None:
+ return context.run(lambda: _run(team, query, variables, timeout=timeout, start_by=start_by))
+
+
+class ProjectChecks:
+ """The checks for one generation. The profile comes from `attribute_profile`, or is None when it failed."""
+
+ def __init__(
+ self, team: Team, metric_names: Sequence[str], value_counts: Mapping[str, int] | None, *, deadline: float
+ ) -> None:
+ self._team = team
+ self._metric_names = tuple(metric_names)
+ self._value_counts = value_counts
+ self._deadline = deadline
+
+ def knows(self, attribute: str) -> bool:
+ return self._value_counts is not None and _attribute(attribute) in self._value_counts
+
+ def has_several_values(self, attribute: str) -> bool:
+ count = self._value_counts.get(_attribute(attribute)) if self._value_counts is not None else None
+ return count is None or count >= MIN_VARIABLE_VALUES
+
+ def one_to_one(self, first: str, second: str) -> bool:
+ if _attribute(first) == _attribute(second):
+ return True
+ try:
+ return one_to_one(self._team, self._metric_names, first, second)
+ except Exception:
+ logger.warning("metrics_suggested_dashboards_variable_pair_failed", team_id=self._team.id)
+ return False
+
+ def run_errors(self, queries: Mapping[str, dict[str, Any]], variables: Sequence[dict[str, Any]]) -> dict[str, str]:
+ """The error of each query that fails, with every variable set to All.
+
+ A run counts as too slow when it takes longer than the deadline from its own start, so a panel that waits
+ for a worker is not held against it. Every run finishes before this returns, so that no query of this
+ generation runs on after it. ClickHouse ends each statement of a metrics query after 30 seconds.
+ """
+ definitions = [MetricsDashboardVariable.model_validate(variable) for variable in variables]
+ start_by = time.monotonic() + RUN_BUDGET_SECONDS
+ errors: dict[str, str] = {}
+ with ThreadPoolExecutor(max_workers=RUN_WORKERS, thread_name_prefix="metrics-suggested-check") as executor:
+ futures = {
+ # Each run keeps the query tags of the activity, in its own copy of the context.
+ key: executor.submit(
+ _run_in,
+ contextvars.copy_context(),
+ self._team,
+ query,
+ definitions,
+ timeout=self._deadline,
+ start_by=start_by,
+ )
+ for key, query in queries.items()
+ }
+ wait(futures.values())
+ for key, future in futures.items():
+ if (error := future.result()) is not None:
+ errors[key] = error
+ return errors
diff --git a/products/metrics/backend/suggested_dashboards/generation.py b/products/metrics/backend/suggested_dashboards/generation.py
index 65788a891b88..3bdd6ca45c74 100644
--- a/products/metrics/backend/suggested_dashboards/generation.py
+++ b/products/metrics/backend/suggested_dashboards/generation.py
@@ -40,6 +40,7 @@ from products.metrics.backend.metric_attributes_query_runner import (
)
from products.metrics.backend.models import MetricsDashboardTemplate
from products.metrics.backend.suggested_dashboards import prompts
+from products.metrics.backend.suggested_dashboards.checks import ProjectChecks, attribute_profile, variable_candidates
from products.metrics.backend.suggested_dashboards.dashboards import (
SERVICE_ATTRIBUTES,
acting_user,
@@ -60,7 +61,13 @@ from products.metrics.backend.suggested_dashboards.spec import (
template_panels,
)
from products.metrics.backend.suggested_dashboards.timeouts import GENERATE_TIMEOUT
-from products.metrics.backend.variables import RANGE_VARIABLES, VARIABLE_NAME, referenced_names
+from products.metrics.backend.variables import (
+ RANGE_VARIABLES,
+ REFERENCE,
+ VARIABLE_NAME,
+ reference_name,
+ referenced_names,
+)
logger = structlog.get_logger(__name__)
@@ -187,34 +194,83 @@ class _Converted:
accepted_variables: tuple[DraftVariable, ...] = ()
-def _valid_variables(drafts: Sequence[DraftVariable], attributes: Mapping[str, Sequence[str]]) -> list[DraftVariable]:
- """The drafted variables that list a known attribute, with unique names."""
+@frozen
+class _ValidVariables:
+ variables: list[DraftVariable]
+ # Each variable that selects the same thing as a kept one, by name, with that kept variable.
+ replaced: dict[str, tuple[DraftVariable, DraftVariable]]
+
+
+def _valid_variables(
+ drafts: Sequence[DraftVariable], attributes: Mapping[str, Sequence[str]], checks: ProjectChecks | None
+) -> _ValidVariables:
+ """The drafted variables that list a known attribute with several values, with unique names.
+
+ A variable whose values go one to one with those of an earlier variable selects the same thing, so the earlier
+ one replaces it.
+ """
known = {key for keys in attributes.values() for key in keys} | SERVICE_ATTRIBUTES
valid: list[DraftVariable] = []
+ replaced: dict[str, tuple[DraftVariable, DraftVariable]] = {}
for draft in drafts:
if len(valid) >= MAX_VARIABLES:
break
if (
not VARIABLE_NAME.match(draft.name)
or draft.name in RANGE_VARIABLES
- or draft.attribute not in known
+ or (draft.attribute not in known and not (checks is not None and checks.knows(draft.attribute)))
or any(variable.name == draft.name for variable in valid)
):
continue
+ if checks is not None and not checks.has_several_values(draft.attribute):
+ continue
+ equivalent = (
+ next((variable for variable in valid if checks.one_to_one(variable.attribute, draft.attribute)), None)
+ if checks is not None
+ else None
+ )
+ if equivalent is not None:
+ replaced[draft.name] = (draft, equivalent)
+ continue
valid.append(draft)
- return valid
+ return _ValidVariables(variables=valid, replaced=replaced)
-def _with_known_references(draft: DraftPanel, names: set[str]) -> DraftPanel:
- """The panel without the filters that refer to a variable that the dashboard does not have."""
+def _same_attribute(first: str, second: str) -> bool:
+ return first == second or {first, second} <= SERVICE_ATTRIBUTES
- def known(builder_filter: BuilderFilter) -> bool:
- return set(referenced_names(builder_filter.value)) <= names
- clauses = [
- clause.model_copy(update={"filters": [item for item in clause.filters if known(item)]})
- for clause in draft.query.clauses
- ]
+def _replace_reference(builder_filter: BuilderFilter, old: DraftVariable, new: DraftVariable) -> BuilderFilter:
+ value = REFERENCE.sub(
+ lambda match: (
+ match.group(0).replace(old.name, new.name, 1) if reference_name(match) == old.name else match.group(0)
+ ),
+ builder_filter.value,
+ )
+ key = new.attribute if _same_attribute(builder_filter.key, old.attribute) else builder_filter.key
+ return builder_filter.model_copy(update={"key": key, "value": value})
+
+
+def _with_known_references(draft: DraftPanel, valid: _ValidVariables) -> DraftPanel:
+ """The panel without the filters that refer to a variable that the dashboard does not have.
+
+ A filter on a replaced variable filters on the variable that replaced it.
+ """
+ names = {variable.name for variable in valid.variables}
+
+ def rewritten(builder_filter: BuilderFilter) -> BuilderFilter:
+ for name in referenced_names(builder_filter.value):
+ if name in valid.replaced:
+ builder_filter = _replace_reference(builder_filter, *valid.replaced[name])
+ return builder_filter
+
+ clauses = []
+ for clause in draft.query.clauses:
+ filters: list[BuilderFilter] = []
+ for item in map(rewritten, clause.filters):
+ if set(referenced_names(item.value)) <= names and item not in filters:
+ filters.append(item)
+ clauses.append(clause.model_copy(update={"filters": filters}))
return draft.model_copy(update={"query": draft.query.model_copy(update={"clauses": clauses})})
@@ -225,22 +281,34 @@ def _convert(
*,
variables: Sequence[DraftVariable] = (),
attributes: Mapping[str, Sequence[str]] | None = None,
+ checks: ProjectChecks | None = None,
) -> _Converted:
"""Check each drafted query as PostHog would run it, and keep the panels that pass, with no overlapping boxes.
A drafted variable stays when it lists a known attribute and a kept panel refers to it. It selects All, because
- the template is for every project and a value of the source project must not reach the others.
+ the template is for every project and a value of the source project must not reach the others. With `checks`,
+ each panel also runs in the source project, and a panel whose run fails goes.
"""
- valid_variables = _valid_variables(variables, attributes or {})
- drafts = [_with_known_references(draft, {variable.name for variable in valid_variables}) for draft in drafts]
+ valid = _valid_variables(variables, attributes or {}, checks)
+ valid_variables = valid.variables
+ definitions = {
+ variable.name: attribute_variable(
+ name=variable.name,
+ label=" ".join(variable.label.split())[:100],
+ attribute=variable.attribute,
+ multi=variable.multi,
+ )
+ for variable in valid_variables
+ }
+ drafts = [_with_known_references(draft, valid) for draft in drafts]
validator = PanelValidator(team=team, catalog=catalog, promql_available=False)
queries = {draft.key: PanelQuery(language="builder", builder=draft.query) for draft in drafts}
- checks = validator.check_all(queries, deadline_seconds=CHECK_DEADLINE_SECONDS)
+ static_checks = validator.check_all(queries, deadline_seconds=CHECK_DEADLINE_SECONDS)
accepted: list[DraftPanel] = []
queries_by_key: dict[str, dict] = {}
dropped: list[str] = []
for draft in drafts:
- check = checks.get(draft.key)
+ check = static_checks.get(draft.key)
if draft.key in queries_by_key or check is None or not check.ok:
dropped.append(draft.title)
continue
@@ -252,6 +320,12 @@ def _convert(
dropped.append(draft.title)
continue
accepted.append(draft)
+ if checks is not None and accepted:
+ errors = checks.run_errors(
+ {draft.key: queries_by_key[draft.key] for draft in accepted}, list(definitions.values())
+ )
+ dropped.extend(f"{draft.title}: {errors[draft.key]}" for draft in accepted if draft.key in errors)
+ accepted = [draft for draft in accepted if draft.key not in errors]
boxes = place_screenshot_boxes(
[RequestedBox(key=draft.key, layout=draft.layout, min_w=2, min_h=2) for draft in accepted]
)
@@ -271,15 +345,7 @@ def _convert(
panels=panels,
accepted=tuple(accepted),
dropped=tuple(dropped),
- variables=tuple(
- attribute_variable(
- name=variable.name,
- label=" ".join(variable.label.split())[:100],
- attribute=variable.attribute,
- multi=variable.multi,
- )
- for variable in kept_variables
- ),
+ variables=tuple(definitions[variable.name] for variable in kept_variables),
accepted_variables=kept_variables,
)
@@ -292,11 +358,26 @@ def _store_panels(template: MetricsDashboardTemplate, record: GenerationRecord,
record.dropped_panels = list(converted.dropped)
+def _checks(context: _Context, value_counts: Mapping[str, int] | None) -> ProjectChecks:
+ return ProjectChecks(
+ context.team, [entry.name for entry in context.entries], value_counts, deadline=CHECK_DEADLINE_SECONDS
+ )
+
+
def draft_template(template_id: str) -> int:
"""Ask the model for the first version of the panels. Returns how many panels passed the checks."""
context = _context(template_id)
template, record = context.template, _record(context.template)
attributes = _attributes(context.team, context.entries)
+ metric_names = [entry.name for entry in context.entries]
+ try:
+ profile = attribute_profile(context.team, metric_names)
+ except Exception:
+ # Without the profile, the dashboard gets no candidates, and a variable is not checked for its values.
+ logger.warning("metrics_suggested_dashboards_attribute_profile_failed", team_id=context.team.id)
+ profile = None
+ candidates = variable_candidates(profile or {}, len(metric_names))
+ value_counts = {key: stats.value_count for key, stats in profile.items()} if profile is not None else None
draft = ask_model(
team_id=context.team.id,
stage="draft",
@@ -309,6 +390,7 @@ def draft_template(template_id: str) -> int:
entries=context.entries,
attributes=attributes.keys,
attribute_values=attributes.values,
+ variable_candidates=candidates,
)
)
],
@@ -320,12 +402,15 @@ def draft_template(template_id: str) -> int:
context.catalog,
variables=draft.variables,
attributes=attributes.keys,
+ checks=_checks(context, value_counts),
)
template.name = " ".join(draft.name.split())[:200] or template.name
template.description = " ".join(draft.description.split())[:300] or template.description
_store_panels(template, record, converted)
record.attributes = attributes.keys
record.attribute_values = attributes.values
+ record.attribute_value_counts = value_counts
+ record.variable_candidates = candidates
_save_record(template, record, "name", "description", "panels", "variables")
return len(converted.panels)
@@ -429,6 +514,7 @@ def check_preview(template_id: str, round_number: int, *, revise: bool) -> bool:
entries=context.entries,
attributes=record.attributes,
attribute_values=record.attribute_values,
+ variable_candidates=record.variable_candidates,
dropped=record.dropped_panels,
)
),
@@ -445,6 +531,7 @@ def check_preview(template_id: str, round_number: int, *, revise: bool) -> bool:
context.catalog,
variables=critique.variables,
attributes=record.attributes,
+ checks=_checks(context, record.attribute_value_counts),
)
if converted.panels:
current.revised = True
diff --git a/products/metrics/backend/suggested_dashboards/prompts.py b/products/metrics/backend/suggested_dashboards/prompts.py
index 65f6ac776996..27ec9c5c1254 100644
--- a/products/metrics/backend/suggested_dashboards/prompts.py
+++ b/products/metrics/backend/suggested_dashboards/prompts.py
@@ -58,6 +58,9 @@ Variables:
- Add a dashboard variable when the metrics share an attribute that names one of several instances of the system, \
such as a job, an instance, a namespace or a cluster. A user then picks one instance for the whole dashboard. \
service_name, the service that sends the metric, is always available for a variable.
+- variable_candidates lists the attributes that most of the metrics have, with 2 to 20 values. Prefer them. An \
+attribute with one value makes no variable.
+- Two variables never select the same thing, such as a host and its instance id. Keep the one a person knows.
- Add no variable for an attribute that splits one instance, such as a route, a status or a mode. Use group_by \
for those. Use at most 2 variables.
- Every panel whose metrics have the attribute refers to the variable with a filter: key is the attribute, op is \
@@ -92,6 +95,9 @@ Look for problems that a person would notice:
- The layout has holes or overlaps, a half-width panel alone in a row, or stats outside the top row.
- Two panels show the same thing.
- A variable that no panel refers to, or a panel that does not refer to a variable whose attribute its metrics have.
+- A variable whose attribute has one value, or two variables that select the same thing.
+- A panel in panels_that_failed_checks: its query failed, with the error after the title. Fix the query or leave \
+the panel out.
When there is no such problem, set looks_good to true. A small style preference is not a problem.
When there are problems, set looks_good to false, list them, and return the complete corrected panel list and \
@@ -148,12 +154,14 @@ def draft_input(
entries: Sequence[CatalogEntry],
attributes: dict[str, list[str]],
attribute_values: dict[str, list[str]],
+ variable_candidates: Sequence[str],
) -> str:
return json.dumps(
{
"dashboard": {"name": name, "description": description},
"metrics": [{**_entry(entry), "attributes": attributes.get(entry.name, [])} for entry in entries],
"attribute_values": attribute_values,
+ "variable_candidates": list(variable_candidates),
},
indent=1,
)
@@ -166,6 +174,7 @@ def critique_input(
entries: Sequence[CatalogEntry],
attributes: dict[str, list[str]],
attribute_values: dict[str, list[str]],
+ variable_candidates: Sequence[str],
dropped: Sequence[str],
) -> str:
payload: dict[str, Any] = {
@@ -173,6 +182,7 @@ def critique_input(
"variables": [variable.model_dump(mode="json") for variable in variables],
"metric_catalog": [{**_entry(entry), "attributes": attributes.get(entry.name, [])} for entry in entries],
"attribute_values": attribute_values,
+ "variable_candidates": list(variable_candidates),
}
if dropped:
payload["panels_that_failed_checks"] = list(dropped)
diff --git a/products/metrics/backend/suggested_dashboards/spec.py b/products/metrics/backend/suggested_dashboards/spec.py
index 3095e43c3e8e..869419e6b2ce 100644
--- a/products/metrics/backend/suggested_dashboards/spec.py
+++ b/products/metrics/backend/suggested_dashboards/spec.py
@@ -135,5 +135,8 @@ class GenerationRecord(_Model):
draft_variables: list[DraftVariable] = Field(default_factory=list)
attributes: dict[str, list[str]] = Field(default_factory=dict)
attribute_values: dict[str, list[str]] = Field(default_factory=dict)
+ # None when the attribute profile failed, so that a variable is not dropped for a count nobody read.
+ attribute_value_counts: dict[str, int] | None = None
+ variable_candidates: list[str] = Field(default_factory=list)
dropped_panels: list[str] = Field(default_factory=list)
error: str | None = None
diff --git a/products/metrics/backend/tests/test_suggested_dashboards.py b/products/metrics/backend/tests/test_suggested_dashboards.py
index 3fb24fb1247f..ec96d77a912b 100644
--- a/products/metrics/backend/tests/test_suggested_dashboards.py
+++ b/products/metrics/backend/tests/test_suggested_dashboards.py
@@ -20,6 +20,7 @@ from products.metrics.backend.dashboard_import.spec import (
GridLayout,
)
from products.metrics.backend.suggested_dashboards.bank import curated_templates
+from products.metrics.backend.suggested_dashboards.checks import AttributeStats, ProjectChecks, variable_candidates
from products.metrics.backend.suggested_dashboards.dashboards import (
attribute_variable,
build_tiles,
@@ -66,6 +67,24 @@ def _panel(
return TemplatePanel(key=key, title=key, query=_query(*names, kind=kind), layout=GridLayout(x=x, y=y, w=w, h=h))
+class _Checks(ProjectChecks):
+ def __init__(
+ self,
+ value_counts: dict[str, int] | None,
+ pairs: frozenset[frozenset[str]] = frozenset(),
+ errors: dict[str, str] | None = None,
+ ) -> None:
+ super().__init__(Team(id=1), [], value_counts, deadline=0)
+ self._pairs = pairs
+ self._errors = errors or {}
+
+ def one_to_one(self, first: str, second: str) -> bool:
+ return frozenset({first, second}) in self._pairs
+
+ def run_errors(self, queries: Any, variables: Any) -> dict[str, str]:
+ return {key: error for key, error in self._errors.items() if key in queries}
+
+
class TestSuggestedDashboards(SimpleTestCase):
@parameterized.expand([(template.key,) for template in curated_templates()])
def test_bank_template_queries_are_valid_and_fit_the_grid(self, key: str) -> None:
@@ -284,6 +303,116 @@ class TestSuggestedDashboards(SimpleTestCase):
MetricsQueryFilter.model_validate(item) for item in orders["clauses"][0]["filters"]
) == ["instance"]
+ @parameterized.expand(
+ [
+ ("one_value", ["instance"], {"instance": 1}, frozenset(), []),
+ ("several_values", ["instance"], {"instance": 3}, frozenset(), ["instance"]),
+ ("unread_profile", ["instance"], None, frozenset(), ["instance"]),
+ (
+ "one_to_one",
+ ["instance", "host"],
+ {"instance": 3, "host": 3},
+ {frozenset({"instance", "host"})},
+ ["instance"],
+ ),
+ ("independent", ["instance", "host"], {"instance": 3, "host": 3}, frozenset(), ["instance", "host"]),
+ ("profiled_attribute_outside_the_keys", ["env"], {"env": 3}, frozenset(), ["env"]),
+ ]
+ )
+ def test_convert_keeps_variables_with_several_values_that_select_different_things(
+ self,
+ _name: str,
+ attributes: list[str],
+ value_counts: dict[str, int] | None,
+ pairs: frozenset[frozenset[str]],
+ kept: list[str],
+ ) -> None:
+ converted = _convert(
+ [
+ self._draft(
+ "orders",
+ "orders_total",
+ 0,
+ filters=[BuilderFilter(key=attribute, op="eq", value=f"${attribute}") for attribute in attributes],
+ )
+ ],
+ Team(id=1),
+ CATALOG,
+ variables=[DraftVariable(name=attribute, label=attribute, attribute=attribute) for attribute in attributes],
+ attributes={"orders_total": ["instance", "host"]},
+ checks=_Checks(value_counts, frozenset(pairs)),
+ )
+
+ assert [variable["name"] for variable in converted.variables] == kept
+ orders = converted.panels[0].query
+ assert orders is not None
+ assert (
+ filter_referenced_names(
+ MetricsQueryFilter.model_validate(item) for item in orders["clauses"][0].get("filters", [])
+ )
+ == kept
+ )
+
+ def test_convert_filters_on_the_kept_variable_where_a_panel_filtered_on_a_replaced_one(self) -> None:
+ converted = _convert(
+ [
+ self._draft(
+ "orders", "orders_total", 0, filters=[BuilderFilter(key="instance", op="eq", value="$instance")]
+ ),
+ self._draft(
+ "queue", "queue_depth", 4, filters=[BuilderFilter(key="host", op="regex", value="${host:regex}")]
+ ),
+ self._draft(
+ "cache", "orders_total", 8, filters=[BuilderFilter(key="service.name", op="eq", value="$service")]
+ ),
+ ],
+ Team(id=1),
+ CATALOG,
+ variables=[
+ DraftVariable(name="instance", label="Instance", attribute="instance"),
+ DraftVariable(name="host", label="Host", attribute="host"),
+ DraftVariable(name="service", label="Service", attribute="service_name"),
+ ],
+ attributes={"orders_total": ["instance", "host"]},
+ checks=_Checks(
+ {"instance": 3, "host": 3, "service_name": 3},
+ frozenset({frozenset({"instance", "host"}), frozenset({"instance", "service_name"})}),
+ ),
+ )
+
+ assert [variable["name"] for variable in converted.variables] == ["instance"]
+ queue, cache = (panel.query for panel in converted.panels[1:])
+ assert queue is not None and cache is not None
+ assert queue["clauses"][0]["filters"] == [{"key": "instance", "op": "regex", "value": "${instance:regex}"}]
+ assert cache["clauses"][0]["filters"] == [{"key": "instance", "op": "eq", "value": "$instance"}]
+
+ def test_convert_drops_the_panels_whose_query_fails_to_run(self) -> None:
+ converted = _convert(
+ [self._draft("orders", "orders_total", 0), self._draft("queue", "queue_depth", 4)],
+ Team(id=1),
+ CATALOG,
+ checks=_Checks({}, errors={"queue": "Bad regex"}),
+ )
+
+ assert [panel.key for panel in converted.panels] == ["orders"]
+ assert converted.dropped == ("queue: Bad regex",)
+
+ @parameterized.expand(
+ [
+ ("one_value", 1, 3, False),
+ ("few_values_on_most_metrics", 5, 3, True),
+ ("on_half_of_the_metrics", 5, 2, False),
+ ("too_many_values", 21, 4, False),
+ ("at_the_limit", 20, 4, True),
+ ]
+ )
+ def test_variable_candidates_have_a_few_values_on_most_metrics(
+ self, _name: str, value_count: int, metric_count: int, candidate: bool
+ ) -> None:
+ profile = {"env": AttributeStats(value_count=value_count, metric_count=metric_count)}
+
+ assert (variable_candidates(profile, 4) == ["env"]) is candidate
+
@staticmethod
def _draft(key: str, metric: str, x: int, filters: list[BuilderFilter] | None = None) -> DraftPanel:
return DraftPanel(
diff --git a/products/metrics/backend/tests/test_suggested_dashboards_checks.py b/products/metrics/backend/tests/test_suggested_dashboards_checks.py
new file mode 100644
index 000000000000..8ac55e522cc5
--- /dev/null
+++ b/products/metrics/backend/tests/test_suggested_dashboards_checks.py
@@ -0,0 +1,103 @@
+import datetime as dt
+
+import pytest
+from posthog.test.base import APIBaseTest, ClickhouseTestMixin
+from unittest.mock import patch
+
+from clickhouse_driver.errors import NetworkError
+
+from posthog.clickhouse.query_tagging import get_query_tag_value, tags_context
+
+from products.metrics.backend.hogql_queries.metrics_query_runner import MetricsQueryRunner
+from products.metrics.backend.suggested_dashboards.checks import (
+ ProjectChecks,
+ _run,
+ attribute_profile,
+ one_to_one,
+ variable_candidates,
+)
+from products.metrics.backend.suggested_dashboards.dashboards import attribute_variable
+from products.metrics.backend.tests._seeder import seed_metric, truncate_metrics_tables
+
+METRICS = ["orders_total", "queue_depth", "cache_hits"]
+
+
+class TestSuggestedDashboardChecks(ClickhouseTestMixin, APIBaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ truncate_metrics_tables()
+ now = dt.datetime.now(dt.UTC) - dt.timedelta(minutes=5)
+ for metric, labels in [
+ ("orders_total", {"env": "prod", "host": "h1", "instance": "i1", "zone": "z1"}),
+ ("orders_total", {"env": "prod", "host": "h2", "instance": "i2", "zone": "z2"}),
+ ("orders_total", {"env": "dev", "host": "h3", "instance": "i3", "zone": "z3"}),
+ ("queue_depth", {"env": "dev", "host": "h1", "instance": "i1", "zone": "z1"}),
+ ("cache_hits", {"region": "eu", "zone": "z4"}),
+ ]:
+ seed_metric(team_id=self.team.id, metric_name=metric, points=[(now, 1.0)], labels=labels)
+
+ def test_profile_counts_values_and_metrics_of_each_attribute(self) -> None:
+ profile = attribute_profile(self.team, METRICS)
+
+ assert {key: (stats.value_count, stats.metric_count) for key, stats in profile.items()} == {
+ "env": (2, 2),
+ "host": (3, 2),
+ "instance": (3, 2),
+ "region": (1, 1),
+ "zone": (4, 3),
+ "service_name": (1, 3),
+ }
+ assert variable_candidates(profile, len(METRICS)) == ["zone", "env", "host", "instance"]
+
+ def test_one_to_one_finds_attributes_that_select_the_same_series(self) -> None:
+ assert one_to_one(self.team, METRICS, "host", "instance")
+ assert not one_to_one(self.team, METRICS, "env", "host")
+ # Only zone selects the cache_hits series.
+ assert not one_to_one(self.team, METRICS, "host", "zone")
+
+ def test_run_errors_reports_only_the_queries_that_fail(self) -> None:
+ def query(op: str, value: str) -> dict:
+ return {
+ "kind": "MetricsQuery",
+ "clauses": [
+ {
+ "name": "a",
+ "metricName": "queue_depth",
+ "aggregation": "avg",
+ "filters": [{"key": "host", "op": op, "value": value}],
+ }
+ ],
+ }
+
+ checks = ProjectChecks(self.team, METRICS, None, deadline=30)
+ errors = checks.run_errors(
+ {"fine": query("eq", "$host"), "broken": query("eq", "$missing")},
+ [attribute_variable(name="host", label="Host", attribute="host")],
+ )
+
+ assert list(errors) == ["broken"]
+
+ def test_a_transient_failure_propagates_and_a_late_run_leaves_the_panel_unchecked(self) -> None:
+ query = {"kind": "MetricsQuery", "clauses": [{"name": "a", "metricName": "queue_depth", "aggregation": "avg"}]}
+ checks = ProjectChecks(self.team, METRICS, None, deadline=30)
+
+ with (
+ patch.object(MetricsQueryRunner, "calculate", side_effect=NetworkError("connection reset")),
+ pytest.raises(NetworkError),
+ ):
+ checks.run_errors({"a": query}, [])
+ assert _run(self.team, {"kind": "MetricsQuery", "clauses": []}, [], timeout=30, start_by=0) is None
+
+ def test_each_run_keeps_the_query_tags_of_the_caller(self) -> None:
+ query = {"kind": "MetricsQuery", "clauses": [{"name": "a", "metricName": "queue_depth", "aggregation": "avg"}]}
+ seen: list[object] = []
+
+ with (
+ tags_context(team_id=4242),
+ patch.object(
+ MetricsQueryRunner, "calculate", side_effect=lambda: seen.append(get_query_tag_value("team_id"))
+ ),
+ ):
+ ProjectChecks(self.team, METRICS, None, deadline=30).run_errors({"a": query, "b": query}, [])
+
+ assert seen == [4242, 4242]