:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(metrics): keep grafana variables live on imported dashboards (#118637)
Frank Hamandeb99cb43b5e7
diff --git a/products/dashboards/backend/dashboard_creation.py b/products/dashboards/backend/dashboard_creation.pyindex d6412f44afca..aab3efbb855b 100644--- a/products/dashboards/backend/dashboard_creation.py+++ b/products/dashboards/backend/dashboard_creation.py@@ -75,6 +75,7 @@ class NewDashboard:# An unlisted dashboard stays out of the dashboard list, and its insights stay out of the saved insights list.# The product that created it opens it by id.unlisted: bool = False+ filters: dict[str, Any] | None = None@frozen@@ -132,7 +133,10 @@ def create_dashboard_with_tiles(*, team_id: int, user_id: int, dashboard: NewDasname=dashboard.name[:MAX_DASHBOARD_NAME_LENGTH],description=dashboard.description,created_by=user,- filters={"date_from": dashboard.date_from} if dashboard.date_from else {},+ filters={+ **(dashboard.filters or {}),+ **({"date_from": dashboard.date_from} if dashboard.date_from else {}),+ },creation_mode=Dashboard.CreationMode.UNLISTED if dashboard.unlisted else Dashboard.CreationMode.DEFAULT,)for index, tile in enumerate(dashboard.tiles):diff --git a/products/metrics/backend/dashboard_import/grafana.py b/products/metrics/backend/dashboard_import/grafana.pyindex 0129808f49f8..7fb6e520392b 100644--- a/products/metrics/backend/dashboard_import/grafana.py+++ b/products/metrics/backend/dashboard_import/grafana.py@@ -196,7 +196,7 @@ class GrafanaDashboardParser:for item in templatingif isinstance(item, dict) and item.get("type") == "datasource" and isinstance(item.get("query"), str)}- self._variables = TemplateVariables.from_templating(templating)+ self._variables = TemplateVariables.from_templating(templating, self._datasource_type)self._has_adhoc_filters = any(isinstance(item, dict) and item.get("type") == "adhoc" for item in templating)self._used_keys: set[str] = set()@@ -242,6 +242,7 @@ class GrafanaDashboardParser:date_from=date_from,panels=panels,variables=self._variables.summary(),+ metric_variables=[definition.model_dump(exclude_none=True) for definition in self._variables.definitions()],notes=notes,)@@ -518,10 +519,14 @@ class GrafanaDashboardParser:notes.append(f"Query {ref_id} is too long to import.")continuelanguage = self._language(datasource, target, expression, grafana_type)- substituted, unresolved = self._variables.substitute(expression)+ substitution = self._variables.substitute(expression)+ substituted, unresolved = substitution.text, list(substitution.unresolved)+ templated = self._variables.template(expression) if language == "promql" else Noneif language == "promql":rewrite = strip_grafana_ranges(substituted)substituted = drop_match_all_matchers(rewrite.expr)+ if templated is not None:+ templated = drop_match_all_matchers(strip_grafana_ranges(templated).expr)if rewrite.changed_meaning:notes.append("A $__range window shows per-step values, not one total for the dashboard range.")notes.extend(f"The variable ${name} has no saved value." for name in unresolved)@@ -535,6 +540,7 @@ class GrafanaDashboardParser:original_expr=expression,legend=legend if isinstance(legend, str) and legend else None,unresolved_variables=unresolved,+ templated_expr=templated,))if hidden:diff --git a/products/metrics/backend/dashboard_import/grafana_v2.py b/products/metrics/backend/dashboard_import/grafana_v2.pyindex 3f9305302106..7bc0445c6b47 100644--- a/products/metrics/backend/dashboard_import/grafana_v2.py+++ b/products/metrics/backend/dashboard_import/grafana_v2.py@@ -65,10 +65,19 @@ def _variable(item: Any) -> dict[str, Any] | None:if kind is None or not isinstance(spec.get("name"), str):return Nonevariable = {**spec, "type": kind}+ query = spec.get("query")if kind == "datasource":# A classic datasource variable names its plugin in `query`.variable["query"] = spec.get("pluginId")- elif not isinstance(spec.get("query"), str):+ elif isinstance(query, dict):+ # A query variable wraps its query as a data query, which names its plugin as a panel query does.+ plugin = query.get("group") if query.get("kind") == "DataQuery" else query.get("kind")+ variable["query"] = _dict(query.get("spec"))+ variable["datasource"] = {+ "type": _dict(spec.get("datasource")).get("type") or plugin,+ "uid": _dict(spec.get("datasource")).get("uid") or _dict(query.get("datasource")).get("name"),+ }+ elif not isinstance(query, str):variable.pop("query", None)return variablediff --git a/products/metrics/backend/dashboard_import/grafana_variables.py b/products/metrics/backend/dashboard_import/grafana_variables.pyindex bf5ce3a31bb5..a408d63ae044 100644--- a/products/metrics/backend/dashboard_import/grafana_variables.py+++ b/products/metrics/backend/dashboard_import/grafana_variables.py@@ -3,28 +3,47 @@from __future__ import annotationsimport re+from collections import Counter+from collections.abc import Callablefrom typing import Any-from posthog.dataclasses import frozen+import re2-_REFERENCE = re.compile(- r"\$\{(?P<braced>[A-Za-z0-9_]+)(?::(?P<braced_format>[A-Za-z0-9_]+))?\}"- r"|\[\[(?P<bracketed>[A-Za-z0-9_]+)(?::(?P<bracketed_format>[A-Za-z0-9_]+))?\]\]"- r"|\$(?P<plain>[A-Za-z0-9_]+)"+from posthog.schema import MetricsDashboardVariable, MetricsDashboardVariableSelection, MetricsDashboardVariableSource++from products.metrics.backend.variable_options import re2_pattern+from products.metrics.backend.variables import (+ REFERENCE,+ VARIABLE_NAME,+ Substitution,+ Variable,+ reference_name,+ referenced_names,+ regex_escape,+ substitute_promql,)-# The range rewrite handles these, so the substitution must leave them in place.-GRAFANA_RANGE_VARIABLES = frozenset({"__rate_interval", "__interval", "__range", "__range_s", "__range_ms", "__auto"})-_REGEX_SPECIAL = set("\\^$.|?*+()[]{}")+_SKIPPED_TYPES = frozenset({"datasource", "adhoc", "groupby"})ALL_VALUE = "$__all"---@frozen-class Variable:- values: tuple[str, ...]- is_all: bool- all_value: str | None- options: tuple[str, ...]+_PROMQL_VARIABLE_QUERY = re.compile(r"^\s*(label_values|label_names|metrics|query_result)\s*\(", re.IGNORECASE)+# Macros such as $__rate_interval, which only resolve inside a panel.+_RANGE_MACRO = re.compile(r"\$\{?__|\[\[__")+# The limits of the variable options endpoint, so that each live variable can load its options.+MAX_NAME_LENGTH = 128+MAX_QUERY_LENGTH = 10_000+MAX_VALUE_LENGTH = 1024+MAX_VALUES = 1000+# Each change to a selection walks the variables that depend on it, so a dashboard keeps a practical number live.+MAX_LIVE_VARIABLES = 50+# An item holds `text : value`, so it can be longer than a value, but not much.+_MAX_CUSTOM_ITEM_LENGTH = 4 * MAX_VALUE_LENGTH+_CUSTOM_SEPARATOR = re.compile(r"\s:\s")+_REGEX_SYNTAX = re.compile(r"[\\^$.|?*+()\[\]{}]")+# Private use characters, which stand for variables while dependencies are checked.+_MARKER_START = 0xF0000+_MAX_MARKERS = 0xFFFD+# The datasources whose variable queries the PromQL options endpoint can run. None is the default datasource.+_PROMQL_DATASOURCES = frozenset({None, "prometheus"})def _strings(value: Any) -> tuple[str, ...]:@@ -37,18 +56,247 @@ def _strings(value: Any) -> tuple[str, ...]:return ()-def regex_escape(value: str) -> str:- # The value lands inside a double-quoted PromQL string, so each regex escape needs a doubled backslash.- return "".join(f"\\\\{char}" if char in _REGEX_SPECIAL else char for char in value)+def _query_text(item: dict[str, Any]) -> str:+ query = item.get("query")+ if isinstance(query, dict):+ query = query.get("query")+ return query.strip() if isinstance(query, str) else ""+++def _regex(item: dict[str, Any]) -> str | None:+ regex = item.get("regex")+ if not isinstance(regex, str) or not regex.strip():+ return None+ return regex.strip()+++def _custom_items(query: str) -> list[str] | None:+ """The comma-separated items of a custom list, where `\\,` is a literal comma. None when the list is too large."""+ items: list[str] = []+ item_start = search_from = 0+ while True:+ comma = query.find(",", search_from)+ end = len(query) if comma < 0 else comma+ if end - item_start > _MAX_CUSTOM_ITEM_LENGTH or len(items) > MAX_VALUES:+ return None+ if comma > 0 and query[comma - 1] == "\\":+ search_from = comma + 1+ continue+ items.append(query[item_start:end])+ if comma < 0:+ return items+ item_start = search_from = comma + 1+++def _custom_values(query: str) -> list[str] | None:+ """The values of a Grafana custom list. An item is `value` or `text : value`. None when the list is too large."""+ items = _custom_items(query)+ if items is None:+ return None+ values = []+ for item in items:+ text = item.replace("\\,", ",").strip()+ # Grafana splits `text : value` at the last separator that has text on both sides.+ separators = [match.start() for match in _CUSTOM_SEPARATOR.finditer(text) if 0 < match.start() < len(text) - 3]+ value = (text[separators[-1] + 3 :] if separators else text).strip()+ if value:+ values.append(value)+ return values+++def _within_limits(values: list[str]) -> bool:+ return len(values) <= MAX_VALUES and all(len(value) <= MAX_VALUE_LENGTH for value in values)+++def _runs_live(regex: str | None) -> bool:+ """Whether the options endpoint runs the regex as Grafana does."""+ if regex is None:+ return True+ # The options endpoint does not put variable values into the regex.+ if len(regex) > MAX_VALUE_LENGTH or referenced_names(regex):+ return False+ # The options endpoint takes one match per option, so a flag such as `g`, which Grafana uses to take every match,+ # changes the values.+ if regex.startswith("/") and regex.rfind("/") > 0 and set(regex[regex.rfind("/") + 1 :]) - set("ims"):+ return False+ try:+ pattern = re2.compile(re2_pattern(regex))+ except re2.error:+ return False+ # Grafana reads the `text` and `value` groups by name. The options endpoint takes the first group.+ return not pattern.groupindex+++def _live_definition(+ name: str, kind: Any, item: dict[str, Any], variable: Variable, datasource_type: str | None+) -> MetricsDashboardVariable | None:+ """The dashboard variable that keeps a Grafana variable live. None when it cannot stay live."""+ if not VARIABLE_NAME.match(name) or variable.default_format is not None:+ return None+ if len(name) > MAX_NAME_LENGTH:+ return None+ custom_values: list[str] | None = None+ query: str | None = None+ all_value = variable.all_value+ # A live all value is regex text that gets the escapes of its string. Grafana's text already has the escapes of the+ # string it was written for, and with a backslash that differs between a quoted and a backtick string.+ if all_value is not None and "\\" in all_value:+ return None+ include_all = bool(item.get("includeAll"))+ if kind == "query":+ query = _query_text(item)+ if not _PROMQL_VARIABLE_QUERY.match(query) or _RANGE_MACRO.search(query) or len(query) > MAX_QUERY_LENGTH:+ return None+ # Grafana runs the regex in JavaScript, and the options endpoint runs it with RE2, which has no lookaround.+ if datasource_type not in _PROMQL_DATASOURCES or not _runs_live(_regex(item)):+ return None+ # A live All without an allValue matches every value, so it must not drop a filter that limits the options.+ if include_all and all_value is None and ("{" in query or _regex(item)):+ return None+ source = MetricsDashboardVariableSource.PROMQL+ elif kind == "custom":+ parsed = [] if variable.options else _custom_values(_query_text(item))+ if parsed is None:+ return None+ custom_values = list(variable.options) or parsed or list(variable.values)+ if not _within_limits(custom_values):+ return None+ if include_all and all_value is None:+ # Grafana matches only the listed values. A value with regex syntax would need escapes for each context.+ if not custom_values or any(regex_escape(value) != value for value in custom_values):+ return None+ all_value = f"({'|'.join(custom_values)})"+ source = MetricsDashboardVariableSource.CUSTOM+ elif kind in ("textbox", "constant"):+ source = MetricsDashboardVariableSource.TEXTBOX+ else:+ return None+ selected = list(variable.options if variable.is_all and not variable.values else variable.values)+ if not _within_limits(selected) or (all_value is not None and len(all_value) > MAX_VALUE_LENGTH):+ return None+ label = item.get("label")+ return MetricsDashboardVariable(+ name=name,+ label=label if isinstance(label, str) and label else None,+ source=source,+ customValues=custom_values,+ query=query,+ regex=_regex(item) if kind == "query" else None,+ multi=bool(item.get("multi")) or None,+ includeAll=include_all or None,+ allValue=all_value,+ current=MetricsDashboardVariableSelection(+ values=selected,+ all=variable.is_all or None,+ ),+ )+++def _size_bound(variable: Variable) -> int:+ """The most characters that a reference to the variable becomes, with the regex alternation.++ A regex escape doubles a character, and the string escape doubles that backslash again, so `.` becomes `\\\\.`.+ """+ if variable.is_all and variable.all_value is not None:+ parts: tuple[str, ...] = (variable.all_value,)+ else:+ parts = variable.options if variable.is_all else variable.values+ return 4 * sum(len(part) for part in parts) + len(parts) + 2+++def _in_loops(parents: dict[str, set[str]]) -> set[str]:+ """The variables in a loop of queries, which the dashboard cannot load in order."""++ def peel(edges: dict[str, set[str]]) -> set[str]:+ # Remove the variables that wait on nothing left, until only the loops and what they block stay.+ waiting = {name: len(names) for name, names in edges.items()}+ blocked: dict[str, list[str]] = {}+ for name, names in edges.items():+ for other in names:+ blocked.setdefault(other, []).append(name)+ ready = [name for name, count in waiting.items() if count == 0]+ while ready:+ for name in blocked.get(ready.pop(), ()):+ waiting[name] -= 1+ if waiting[name] == 0:+ ready.append(name)+ return {name for name, count in waiting.items() if count > 0}++ after_loops = peel(parents)+ # The same pass over the reversed graph leaves out the variables that only depend on a loop.+ children: dict[str, set[str]] = {name: set() for name in after_loops}+ for name in after_loops:+ for parent in parents[name] & after_loops:+ children[parent].add(name)+ return peel(children)+++def _resolvable(+ live: dict[str, MetricsDashboardVariable], variables: dict[str, Variable]+) -> dict[str, MetricsDashboardVariable]:+ """The live variables whose queries refer only to live variables or to variables with a saved value.++ A query that the saved values of static variables would make longer than the options API accepts is static too,+ and so is a variable in a loop of queries.+ """+ valued = [name for name, variable in variables.items() if variable.values or variable.is_all]+ if len(valued) > _MAX_MARKERS:+ return {}+ # Each variable with a value becomes one marker character, so a query is inspected without its saved values.+ markers = {chr(_MARKER_START + index): name for index, name in enumerate(valued)}+ stand_ins = {+ name: Variable(values=(marker,), is_all=False, all_value=None, options=()) for marker, name in markers.items()+ }+ hard: dict[str, list[str]] = {}+ soft: dict[str, list[str]] = {}+ references: dict[str, Counter[str]] = {}+ size: dict[str, int] = {}+ bounds = {name: _size_bound(variable) for name, variable in variables.items()}+ static: list[str] = []+ parents: dict[str, set[str]] = {}+ for name, definition in live.items():+ substitution = substitute_promql(definition.query or "", stand_ins)+ references[name] = Counter(markers[char] for char in substitution.text if char in markers)+ parents[name] = {other for other in [*substitution.unresolved, *references[name]] if other in live}+ size[name] = len(definition.query or "") + sum(+ count * bounds[other] for other, count in references[name].items() if other not in live+ )+ if any(other not in live for other in substitution.unresolved) or size[name] > MAX_QUERY_LENGTH:+ static.append(name)+ for other in substitution.unresolved:+ hard.setdefault(other, []).append(name)+ for other in references[name]:+ if other in live:+ soft.setdefault(other, []).append(name)+ static.extend(_in_loops(parents))+ # Each variable becomes static at most once, so the work grows with the number of references.+ removed: set[str] = set()+ while static:+ name = static.pop()+ if name in removed:+ continue+ removed.add(name)+ static.extend(hard.get(name, ()))+ # A variable that becomes static puts its saved values into the queries that refer to it.+ for dependent in soft.get(name, ()):+ size[dependent] += references[dependent][name] * bounds[name]+ if size[dependent] > MAX_QUERY_LENGTH:+ static.append(dependent)+ return {name: definition for name, definition in live.items() if name not in removed}class TemplateVariables:- def __init__(self, variables: dict[str, Variable]) -> None:+ def __init__(self, variables: dict[str, Variable], live: dict[str, MetricsDashboardVariable] | None = None) -> None:self._variables = variables+ self._live = live or {}@classmethod- def from_templating(cls, items: Any) -> TemplateVariables:+ def from_templating(+ cls, items: Any, datasource_type: Callable[[Any], str | None] = lambda _: None+ ) -> TemplateVariables:+ """`datasource_type` gives the type of a Grafana datasource reference, or None when it is unknown."""variables: dict[str, Variable] = {}+ live: dict[str, MetricsDashboardVariable] = {}for item in items if isinstance(items, list) else []:if not isinstance(item, dict) or not isinstance(item.get("name"), str):continue@@ -70,53 +318,64 @@ class TemplateVariables:)all_value = item.get("allValue") if isinstance(item.get("allValue"), str) else Noneis_all = ALL_VALUE in values or (current.get("text") == "All" and bool(item.get("includeAll")))- variables[item["name"]] = Variable(- values=tuple(value for value in values if value != ALL_VALUE),+ selected = tuple(value for value in values if value != ALL_VALUE)+ # Grafana puts the value of a text box into a regex matcher as it is, so it can hold a regex.+ as_written = kind in ("textbox", "constant") and any(_REGEX_SYNTAX.search(value) for value in selected)+ variable = Variable(+ values=selected,is_all=is_all,all_value=all_value or None,options=options,+ default_format="raw" if as_written else None,+ all_value_as_written=True,)- return cls(variables)+ variables[item["name"]] = variable+ # Checking a definition compiles its regex, so the variables past the cap are not checked.+ if len(live) < MAX_LIVE_VARIABLES:+ definition = _live_definition(+ item["name"], kind, item, variable, datasource_type(item.get("datasource"))+ )+ if definition is not None:+ live[item["name"]] = definition+ return cls(variables, _resolvable(live, variables))def summary(self) -> dict[str, str]:return {name: "All" if variable.is_all else ", ".join(variable.values) for name, variable in self._variables.items()}- def substitute(self, expr: str) -> tuple[str, list[str]]:+ def definitions(self) -> list[MetricsDashboardVariable]:+ """The variables that the imported dashboard keeps live, in the Grafana order.++ A variable query keeps its references to live variables. The others, such as an interval,+ get their saved values, because the dashboard does not define them.+ """+ return [+ definition.model_copy(+ update={"query": substitute_promql(definition.query, self._variables, keep=self._live).text}+ )+ if definition.query+ else definition+ for definition in self._live.values()+ ]++ def substitute(self, expr: str) -> Substitution:"""Replace variable references with their saved values. Also returns the names it could not resolve."""- unresolved: list[str] = []-- def replace(match: re.Match[str]) -> str:- name = match.group("braced") or match.group("bracketed") or match.group("plain")- value_format = match.group("braced_format") or match.group("bracketed_format")- # A numeric reference such as $1 is a regex group in label_replace, not a variable.- if name in GRAFANA_RANGE_VARIABLES or name.isdigit():- return match.group(0)- variable = self._variables.get(name)- if variable is None or (not variable.values and not variable.is_all):- unresolved.append(name)- return match.group(0)- return self._format(variable, value_format)-- return _REFERENCE.sub(replace, expr), list(dict.fromkeys(unresolved))-- @staticmethod- def _format(variable: Variable, value_format: str | None) -> str:- if variable.is_all:- if variable.all_value is not None:- return variable.all_value- if not variable.options:- return ".*"- values = variable.options- value_format = value_format or "regex"- else:- values = variable.values- if value_format in ("raw", "text", "csv"):- return ",".join(values)- if value_format == "pipe":- return "|".join(values)- if value_format == "regex" or len(values) > 1:- escaped = [regex_escape(value) for value in values]- return f"({'|'.join(escaped)})" if len(escaped) > 1 else escaped[0]- return values[0]+ return substitute_promql(expr, self._variables)++ def template(self, expr: str) -> str | None:+ """Keep references to live variables and put in the saved values of the others.++ None when the expression refers to no live variable, or to a variable that has no saved value.+ """+ if not any(name in self._live for name in referenced_names(expr)):+ return None+ # A list set to All gets an alternation as its all value, which a format such as `csv` does not change.+ for match in REFERENCE.finditer(expr):+ name = reference_name(match)+ definition = self._live.get(name)+ formatted = match.group("braced_format") or match.group("bracketed_format")+ if formatted and definition is not None and definition.allValue != self._variables[name].all_value:+ return None+ substitution = substitute_promql(expr, self._variables, keep=self._live)+ return None if substitution.unresolved else substitution.textdiff --git a/products/metrics/backend/dashboard_import/importer.py b/products/metrics/backend/dashboard_import/importer.pyindex fbd2798dbc02..ea9b09bb214b 100644--- a/products/metrics/backend/dashboard_import/importer.py+++ b/products/metrics/backend/dashboard_import/importer.py@@ -255,7 +255,7 @@ class PanelResolver:name=panel.title,description=panel.description,query=insight_query(- queries[key],+ self._templated(panel, queries[key]),panel.display,catalog=self._catalog,validator=self._validator,@@ -281,6 +281,17 @@ class PanelResolver:language="promql", promql=combine_targets([(target.ref_id, target.expr) for target in panel.targets]))+ @staticmethod+ def _templated(panel: PanelSpec, query: PanelQuery) -> PanelQuery:+ """The checked query, with the references to live dashboard variables put back.++ The check runs with the saved variable values, because a query with `$name` in it cannot run.+ """+ if query.language != "promql" or not any(target.templated_expr for target in panel.targets):+ return query+ promql = combine_targets([(target.ref_id, target.templated_expr or target.expr) for target in panel.targets])+ return query.model_copy(update={"promql": promql})+def _query_metric_names(queries: Iterable[PanelQuery]) -> list[str]:names: list[str] = []@@ -631,6 +642,9 @@ class DashboardImporter:tiles=tuple(_new_tile(tile) for tile in tiles),idempotency_key=idempotency_key,date_from=state.date_from,+ filters={"metricVariables": state.spec.metric_variables}+ if state.spec and state.spec.metric_variables+ else None,),)except DashboardCreationDenied as error:@@ -730,6 +744,13 @@ class DashboardImporter:key=answer.key, title=title, outcome="failed", reason=error or "The query did not pass the check.")# The parser maps a Grafana panel's display exactly, so only a screenshot panel takes the agent's display.+ query = answer.query+ if spec_panel is not None and any(target.templated_expr for target in spec_panel.targets):+ unchanged = combine_targets([(target.ref_id, target.expr) for target in spec_panel.targets])+ if query.language == "promql" and query.promql == unchanged:+ query = PanelResolver._templated(spec_panel, query)+ else:+ reason = " ".join([reason, "The panel keeps the saved values of the dashboard variables."]).strip()if spec_panel is not None:display = spec_panel.displayelse:@@ -747,7 +768,7 @@ class DashboardImporter:tile=insight_tile(name=title,description=spec_panel.description if spec_panel else "",- query=insight_query(answer.query, display, catalog=catalog, validator=validator, date_from=date_from),+ query=insight_query(query, display, catalog=catalog, validator=validator, date_from=date_from),layout=layout,),)diff --git a/products/metrics/backend/dashboard_import/spec.py b/products/metrics/backend/dashboard_import/spec.pyindex c4082dec5119..761ba8cc7a37 100644--- a/products/metrics/backend/dashboard_import/spec.py+++ b/products/metrics/backend/dashboard_import/spec.py@@ -70,6 +70,8 @@ class TargetSpec(_Model):original_expr: strlegend: str | None = Noneunresolved_variables: list[str] = Field(default_factory=list)+ # The expression with references to live dashboard variables kept. None when it refers to none.+ templated_expr: str | None = Noneclass PanelSpec(_Model):@@ -92,6 +94,8 @@ class DashboardSpec(_Model):date_from: str | None = Nonepanels: list[PanelSpec]variables: dict[str, str] = Field(default_factory=dict)+ # Dashboard variables that stay live on the imported dashboard, as MetricsDashboardVariable JSON.+ metric_variables: list[dict[str, Any]] = Field(default_factory=list)notes: list[str] = Field(default_factory=list)diff --git a/products/metrics/backend/tests/test_dashboard_import_api.py b/products/metrics/backend/tests/test_dashboard_import_api.pyindex 07f98cf31333..e557ac980cd2 100644--- a/products/metrics/backend/tests/test_dashboard_import_api.py+++ b/products/metrics/backend/tests/test_dashboard_import_api.py@@ -33,7 +33,7 @@ CATALOG = []-def _grafana(*exprs: str) -> str:+def _grafana(*exprs: str, templating: list[dict[str, Any]] | None = None) -> str:panels: list[dict[str, Any]] = [{"id": 1, "type": "row", "title": "Orders", "gridPos": {"x": 0, "y": 0, "w": 24, "h": 1}}]@@ -48,7 +48,10 @@ def _grafana(*exprs: str) -> str:"targets": [{"refId": "A", "expr": expr}],})- return json.dumps({"title": "Orders", "time": {"from": "now-6h"}, "panels": panels})+ dashboard: dict[str, Any] = {"title": "Orders", "time": {"from": "now-6h"}, "panels": panels}+ if templating:+ dashboard["templating"] = {"list": templating}+ return json.dumps(dashboard)def _png() -> str:@@ -99,19 +102,48 @@ class TestDashboardImportAPI(APIBaseTest):run.save(update_fields=["status", "output"])delay.assert_called_once_with(self.team.id, import_id)- def test_imports_matching_panels_at_once_without_an_agent(self) -> None:- body = self._start(source="grafana", grafana_json=_grafana("sum(rate(orders_total[$__rate_interval]))"))+ @parameterized.expand(+ [+ ("no_variables", "sum(rate(orders_total[$__rate_interval]))", None, "sum(rate(orders_total))", {}),+ (+ "live_variable",+ 'sum(rate(orders_total{job="$job"}[$__rate_interval]))',+ [{"name": "job", "type": "custom", "query": "api,web", "current": {"text": "api", "value": "api"}}],+ 'sum(rate(orders_total{job="$job"}))',+ {+ "metricVariables": [+ {+ "name": "job",+ "source": "custom",+ "customValues": ["api", "web"],+ "current": {"values": ["api"]},+ }+ ]+ },+ ),+ ]+ )+ def test_imports_matching_panels_at_once_without_an_agent(+ self,+ _name: str,+ expr: str,+ templating: list[dict[str, Any]] | None,+ promql: str,+ variable_filters: dict[str, Any],+ ) -> None:+ body = self._start(source="grafana", grafana_json=_grafana(expr, templating=templating))assert (body["id"], body["status"], body["error"]) == (None, "completed", None)assert body["summary"] == {"total": 2, "imported": 2, "approximated": 0, "failed": 0, "skipped": 0}+ assert "$job" not in self.smoke_run.call_args.args[1]dashboard = Dashboard.objects.get(id=body["dashboard_id"])- assert (dashboard.name, dashboard.filters) == ("Orders", {"date_from": "-6h"})+ assert (dashboard.name, dashboard.filters) == ("Orders", {"date_from": "-6h", **variable_filters})tiles = sorted(DashboardTile.objects.filter(dashboard=dashboard), key=lambda tile: tile.layouts["sm"]["y"])assert tiles[0].text is not None and tiles[0].text.body == "## Orders"assert tiles[1].insight is not None and tiles[1].insight.query == {"kind": "MetricsQuery","language": "promql",- "promql": "sum(rate(orders_total))",+ "promql": promql,"clauses": [],"display": {"type": "line"},}@@ -184,6 +216,29 @@ class TestDashboardImportAPI(APIBaseTest):assert DashboardTile.objects.filter(dashboard_id=first["dashboard_id"]).count() == 3self.storage.delete.assert_called()+ def test_an_agent_panel_without_variable_references_says_so(self) -> None:+ templating = [{"name": "job", "type": "custom", "query": "api,web", "current": {"value": "api"}}]+ started = self._start(+ source="grafana", grafana_json=_grafana('rate(payments_total{job="$job"}[5m])', templating=templating)+ )+ answer = {+ "dashboard_name": "Orders",+ "panels": [+ {+ "key": "p2",+ "title": "Payments",+ "outcome": "approximated",+ "reason": "Uses orders_total.",+ "query": {"language": "promql", "promql": 'sum(rate(orders_total{job="api"}))'},+ }+ ],+ }++ self._finish_run(started["id"], run_status=TaskRun.Status.COMPLETED, output=answer)+ body = self.client.get(f"{self.url}{started['id']}/").json()++ assert "keeps the saved values" in body["panels"][1]["reason"]+@parameterized.expand([("run_failed", TaskRun.Status.FAILED, None, "stopped before it finished"),diff --git a/products/metrics/backend/tests/test_dashboard_import_parsing.py b/products/metrics/backend/tests/test_dashboard_import_parsing.pyindex 9fff58ebc909..41eef9d09bcc 100644--- a/products/metrics/backend/tests/test_dashboard_import_parsing.py+++ b/products/metrics/backend/tests/test_dashboard_import_parsing.py@@ -1,3 +1,4 @@+import timefrom typing import Anyimport pytest@@ -19,7 +20,12 @@ DASHBOARD: dict[str, Any] = {"time": {"from": "now-6h", "to": "now"},"templating": {"list": [- {"name": "job", "type": "query", "current": {"text": "checkout", "value": "checkout"}},+ {+ "name": "job",+ "type": "query",+ "query": {"query": "label_values(up, job)", "refId": "PrometheusVariableQueryEditor-VariableQuery"},+ "current": {"text": "checkout", "value": "checkout"},+ },{"name": "instance","type": "query",@@ -122,6 +128,17 @@ def test_parses_panels_into_queries_displays_and_a_grid_without_overlaps() -> No"p8": "timeseries",}assert panels["p2"].targets[0].expr == 'sum by (code) (rate(orders_total{job="checkout"}))'+ assert panels["p2"].targets[0].templated_expr == 'sum by (code) (rate(orders_total{job="$job"}))'+ assert panels["p3"].targets[0].templated_expr == 'sum(rate(order_errors_total{route=~"$route"}[5m]))'+ assert spec.metric_variables == [+ {+ "name": "job",+ "source": "promql",+ "query": "label_values(up, job)",+ "current": {"values": ["checkout"]},+ },+ {"name": "route", "source": "custom", "customValues": ["/a", "/b.c"], "current": {"values": ["/a", "/b.c"]}},+ ]assert (panels["p2"].display.type, panels["p2"].display.unit) == ("area", "{req}/s")assert panels["p3"].targets[0].expr == 'sum(rate(order_errors_total{route=~"(/a|/b\\\\.c)"}[5m]))'assert (panels["p3"].display.unit, panels["p3"].display.reduce) == ("1", "mean")@@ -306,8 +323,12 @@ def _comparable(raw: dict[str, Any]) -> list[tuple[Any, ...]]:def test_reads_a_v2_resource_like_the_same_classic_dashboard(alpha: bool) -> None:v2 = _v2_dashboard(alpha=alpha)- assert GrafanaDashboardParser(v2).parse().date_from == "-6h"+ spec = GrafanaDashboardParser(v2).parse()+ assert spec.date_from == "-6h"assert _comparable(v2) == _comparable(DASHBOARD)+ assert {variable["name"]: variable.get("query") for variable in spec.metric_variables}["instance"] == (+ "label_values(instance)"+ )def test_places_v2_auto_grid_panels_in_columns_under_each_tab() -> None:@@ -449,7 +470,228 @@ def test_all_without_an_all_value_uses_every_option(expr: str, expected: str) ->}])- assert variables.substitute(expr) == (expected, [])+ substitution = variables.substitute(expr)+ assert (substitution.text, substitution.unresolved) == (expected, ())+++def test_live_variable_queries_get_the_saved_values_of_other_variables() -> None:+ variables = TemplateVariables.from_templating(+ [+ {"name": "window", "type": "interval", "current": {"value": "5m"}},+ {"name": "env", "type": "custom", "query": "prod,dev", "current": {"value": "prod"}},+ {+ "name": "job",+ "type": "query",+ "query": 'query_result(rate(up{env="$env"}[$window]))',+ "current": {"value": "api"},+ },+ ]+ )++ queries = {definition.name: definition.query for definition in variables.definitions()}+ assert queries["job"] == 'query_result(rate(up{env="$env"}[5m]))'+++def test_imported_variable_regex_keeps_its_flags() -> None:+ variables = TemplateVariables.from_templating(+ [{"name": "job", "type": "query", "query": "label_values(job)", "regex": "/api-(.*)/i", "current": {}}]+ )++ assert [definition.regex for definition in variables.definitions()] == ["/api-(.*)/i"]+++ "expr, all_value",+ [('up{job=~"$job"}', "api\\\\..*"), ("up{job=~`$job`}", "api\\..*")],+)+def test_an_all_value_keeps_its_grafana_text(expr: str, all_value: str) -> None:+ variables = TemplateVariables.from_templating(+ [{"name": "job", "type": "custom", "query": "a,b", "allValue": all_value, "current": {"value": "$__all"}}]+ )++ assert variables.substitute(expr).text == expr.replace("$job", all_value)+ assert variables.definitions() == []+++def test_a_panel_that_formats_a_listed_all_keeps_the_saved_values() -> None:+ variables = TemplateVariables.from_templating(+ [{"name": "env", "type": "custom", "query": "prod,dev", "includeAll": True, "current": {"value": "$__all"}}]+ )++ assert variables.template('up{env="${env:csv}"}') is None+ assert variables.template('up{env=~"$env"}') == 'up{env=~"$env"}'+++def test_a_listed_all_matches_only_the_listed_values() -> None:+ variables = TemplateVariables.from_templating(+ [{"name": "env", "type": "custom", "query": "prod,dev", "includeAll": True, "current": {"value": "$__all"}}]+ )++ assert [definition.allValue for definition in variables.definitions()] == ["(prod|dev)"]+++ "item",+ [+ pytest.param(+ {"name": "env", "type": "custom", "query": "a.b,c", "includeAll": True, "current": {"value": "$__all"}},+ id="listed_all_with_regex_syntax",+ ),+ pytest.param(+ {"name": "job", "type": "query", "query": 'label_values(up{env="prod"}, job)', "includeAll": True},+ id="query_all_that_filters_its_options",+ ),+ pytest.param(+ {"name": "job", "type": "query", "query": "label_values(job)", "regex": "api-.*", "includeAll": True},+ id="query_all_with_a_regex",+ ),+ pytest.param(+ {"name": "job", "type": "query", "query": "query_result(rate(up[$__rate_interval]))"},+ id="range_macro",+ ),+ pytest.param(+ {"name": "job", "type": "query", "query": "query_result(" + "x" * 10_000 + ")"},+ id="query_over_the_api_limit",+ ),+ pytest.param({"name": "env", "type": "custom", "query": "a," + "b" * 1025}, id="value_over_the_api_limit"),+ pytest.param({"name": "e" * 129, "type": "textbox", "query": "prod"}, id="name_over_the_api_limit"),+ pytest.param(+ {"name": "job", "type": "query", "query": "label_values(job)", "regex": "/^api-(?!test).*$/"},+ id="regex_that_re2_cannot_run",+ ),+ pytest.param(+ {"name": "pod", "type": "query", "query": 'label_values({app="api"}, pod)', "datasource": "Loki"},+ id="other_datasource",+ ),+ pytest.param({"name": "job", "type": "textbox", "query": "api-.*"}, id="text_box_regex"),+ pytest.param(+ {"name": "job", "type": "query", "query": "label_values(job)", "regex": "/(?<text>[^:]+):(?<value>.+)/"},+ id="named_value_capture",+ ),+ pytest.param(+ {"name": "job", "type": "query", "query": "label_values(job)", "regex": "a" * 1025},+ id="regex_over_the_api_limit",+ ),+ pytest.param(+ {"name": "job", "type": "query", "query": "label_values(job)", "regex": "/([^,]+)/g"},+ id="regex_with_the_global_flag",+ ),+ pytest.param(+ {"name": "job", "type": "query", "query": "label_values(job)", "regex": "/^${env}-(.*)$/"},+ id="regex_with_a_variable",+ ),+ ],+)+def test_variables_that_cannot_stay_live_keep_their_saved_values(item: dict[str, Any]) -> None:+ datasource_type = lambda datasource: "loki" if datasource == "Loki" else None # noqa: E731+ assert TemplateVariables.from_templating([item], datasource_type).definitions() == []+++def test_a_text_box_regex_stays_a_regex() -> None:+ variables = TemplateVariables.from_templating([{"name": "job", "type": "textbox", "query": "api-.*"}])++ assert variables.substitute('up{job=~"$job"}').text == 'up{job=~"api-.*"}'+++def test_a_variable_whose_query_cannot_resolve_stays_static() -> None:+ variables = TemplateVariables.from_templating(+ [+ {"name": "window", "type": "interval", "current": {}},+ {"name": "job", "type": "query", "query": "query_result(rate(up[$window]))", "current": {"value": "a"}},+ {"name": "pod", "type": "query", "query": 'label_values(up{job="$job"}, pod)', "current": {"value": "b"}},+ ]+ )++ assert [(definition.name, definition.query) for definition in variables.definitions()] == [+ ("pod", 'label_values(up{job="a"}, pod)')+ ]+ assert variables.template('up{job="$job"}') is None+++def test_a_query_that_saved_values_would_make_too_long_stays_static() -> None:+ big = {"name": "big", "type": "interval", "current": {"value": ["x" * 1000] * 20}}+ job = {"name": "job", "type": "query", "query": 'label_values(up{a=~"$big"}, b)', "current": {"value": "a"}}++ assert TemplateVariables.from_templating([big, job]).definitions() == []+++def test_the_size_bound_counts_both_escapes() -> None:+ dots = {"name": "dots", "type": "interval", "current": {"value": "." * 1000}}+ query = "label_values(up{" + ", ".join(f'l{index}=~"$dots"' for index in range(4)) + "}, b)"+ job = {"name": "job", "type": "query", "query": query, "current": {"value": "a"}}++ assert TemplateVariables.from_templating([dots, job]).definitions() == []+++def test_a_custom_list_reads_grafana_text_and_value_items() -> None:+ variables = TemplateVariables.from_templating(+ [{"name": "env", "type": "custom", "query": "Production : prod, Development : dev, a\\,b", "current": {}}]+ )++ assert [definition.customValues for definition in variables.definitions()] == [["prod", "dev", "a,b"]]+++def test_a_custom_list_too_large_for_the_api_stays_static() -> None:+ query = "a," + "b" * 5_000_000+ started = time.monotonic()++ variables = TemplateVariables.from_templating([{"name": "env", "type": "custom", "query": query, "current": {}}])++ assert variables.definitions() == []+ assert time.monotonic() - started < 1+++def test_a_long_custom_list_item_parses_in_linear_time() -> None:+ item = "a" + " " * 200_000 + "b"+ started = time.monotonic()++ TemplateVariables.from_templating([{"name": "env", "type": "custom", "query": f"x, {item}, y", "current": {}}])++ assert time.monotonic() - started < 1+++def test_only_the_first_variables_stay_live() -> None:+ items = [{"name": f"v{index}", "type": "custom", "query": "a,b", "current": {"value": "a"}} for index in range(60)]++ assert len(TemplateVariables.from_templating(items).definitions()) == 50+++ "items",+ [+ pytest.param(+ [{"name": "job", "type": "query", "query": 'label_values(up{job="$job"}, job)', "current": {"value": "a"}}],+ id="self_reference",+ ),+ pytest.param(+ [+ {"name": "a", "type": "query", "query": 'label_values(up{b="$b"}, a)', "current": {"value": "x"}},+ {"name": "b", "type": "query", "query": 'label_values(up{a="$a"}, b)', "current": {"value": "y"}},+ ],+ id="two_variable_loop",+ ),+ ],+)+def test_variables_in_a_loop_stay_static(items: list[dict[str, Any]]) -> None:+ parent = items[0]+ query = f'label_values(up{{job="${parent["name"]}"}}, pod)'+ pod = {"name": "pod", "type": "query", "query": query, "current": {"value": "p"}}++ definitions = TemplateVariables.from_templating([*items, pod]).definitions()++ saved = parent["current"]["value"]+ assert [(definition.name, definition.query) for definition in definitions] == [+ ("pod", f'label_values(up{{job="{saved}"}}, pod)')+ ]+++def test_a_long_chain_of_unresolvable_variables_stays_static() -> None:+ chain = [+ {"name": f"v{index}", "type": "query", "query": f'label_values(up{{x="$v{index + 1}"}}, y)'}+ for index in range(2000)+ ]++ assert TemplateVariables.from_templating(chain).definitions() == []@pytest.mark.parametrize(diff --git a/products/metrics/backend/variables.py b/products/metrics/backend/variables.pyindex 74d1175d5b1b..43e3c0f970ce 100644--- a/products/metrics/backend/variables.py+++ b/products/metrics/backend/variables.py@@ -54,6 +54,8 @@ class Variable:all_value_as_written: bool = False# True once All has the values of every option, which can be none.loaded: bool = False+ # The format of a reference that names none, such as `raw` for a Grafana text box that holds a regex.+ default_format: str | None = None@frozen@@ -351,7 +353,8 @@ def substitute_promql(expr: str, variables: Mapping[str, Variable], *, keep: Colunresolved.append(name)return match.group(0)# In a regex matcher, a dot in a selected value such as an IP address must stay a dot.- return format_promql(variable, match.group("braced_format") or match.group("bracketed_format"), context=context)+ value_format = match.group("braced_format") or match.group("bracketed_format") or variable.default_format+ return format_promql(variable, value_format, context=context)return Substitution(text=REFERENCE.sub(replace, expr), unresolved=tuple(dict.fromkeys(unresolved)))