Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(metrics): load the options of dashboard variables (#118634)

Frank Hamandbffc61719422
diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json
index 191e52fa28d0..2de53612353e 100644
--- a/frontend/src/queries/schema.json
+++ b/frontend/src/queries/schema.json
@@ -39553,12 +39553,23 @@
"description": "The value of one variable in a metrics query, set from the dashboard that shows the query.",
"properties": {
"all": {
- "description": "True when the variable is set to All. PromQL gets `allValue` or `.*`, builder filters drop the filter, and SQL gets `values`.",
+ "description": "True when the variable is set to All. PromQL gets `allValue` or `.*`, builder filters drop the filter, and SQL gets every option of the variable.",
"type": "boolean"
},
"allValue": {
"type": ["string", "null"]
},
+ "definition": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/MetricsDashboardVariable"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "The definition of a variable set to All, so that a SQL query can load its options."
+ },
"values": {
"items": {
"type": "string"
diff --git a/frontend/src/queries/schema/schema-general.ts b/frontend/src/queries/schema/schema-general.ts
index cc7a094e88cb..8ebdb84e2353 100644
--- a/frontend/src/queries/schema/schema-general.ts
+++ b/frontend/src/queries/schema/schema-general.ts
@@ -4908,9 +4908,11 @@ export interface MetricsDashboardVariableSelection {
export interface MetricsVariableValue {
values: string[]
/** True when the variable is set to All. PromQL gets `allValue` or `.*`, builder filters drop the
- * filter, and SQL gets `values`. */
+ * filter, and SQL gets every option of the variable. */
all?: boolean
allValue?: string | null
+ /** The definition of a variable set to All, so that a SQL query can load its options. */
+ definition?: MetricsDashboardVariable | null
}
/** A Grafana-style dashboard variable. Metrics tiles refer to it as `$name`, `${name}` or `${name:format}`. */
diff --git a/posthog/schema.py b/posthog/schema.py
index d58b15a2541c..518142dbab81 100644
--- a/posthog/schema.py
+++ b/posthog/schema.py
@@ -2784,21 +2784,6 @@ class MetricsThreshold(BaseModel):
)
-class MetricsVariableValue(BaseModel):
- model_config = ConfigDict(
- extra="forbid",
- )
- all: bool | None = Field(
- default=None,
- description=(
- "True when the variable is set to All. PromQL gets `allValue` or `.*`,"
- " builder filters drop the filter, and SQL gets `values`."
- ),
- )
- allValue: str | None = None
- values: list[str]
-
-
class MetricsYAxisSettings(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -7146,6 +7131,26 @@ class MetricsQueryClause(BaseModel):
)
+class MetricsVariableValue(BaseModel):
+ model_config = ConfigDict(
+ extra="forbid",
+ )
+ all: bool | None = Field(
+ default=None,
+ description=(
+ "True when the variable is set to All. PromQL gets `allValue` or `.*`,"
+ " builder filters drop the filter, and SQL gets every option of the"
+ " variable."
+ ),
+ )
+ allValue: str | None = None
+ definition: MetricsDashboardVariable | None = Field(
+ default=None,
+ description=("The definition of a variable set to All, so that a SQL query can load its options."),
+ )
+ values: list[str]
+
+
class MultiQuestionFormField(BaseModel):
model_config = ConfigDict(
extra="forbid",
diff --git a/posthog/settings/data_stores.py b/posthog/settings/data_stores.py
index 55ff5ce94547..71017c9e99e5 100644
--- a/posthog/settings/data_stores.py
+++ b/posthog/settings/data_stores.py
@@ -753,6 +753,13 @@ CACHES["task_artifact_preview"] = {
"LOCATION": REDIS_URL,
}
+# Metrics dashboard tiles wait for one loader to write the variable options and hold a lock while it runs.
+# A replica that lags would hide the result and the lock, so another tile would run the same query again.
+CACHES["metrics_variable_options"] = {
+ **CACHES["default"],
+ "LOCATION": REDIS_URL,
+}
+
# Dedicated cache for the feature flags service (if configured)
# Django only writes to this cache (never reads), so no reader URL needed
if FLAGS_REDIS_URL:
@@ -819,6 +826,7 @@ if TEST:
CACHES["cohort_dependencies"] = CACHES["default"]
CACHES["ingress_dedup"] = CACHES["default"]
CACHES["task_artifact_preview"] = CACHES["default"]
+ CACHES["metrics_variable_options"] = CACHES["default"]
# Cache timeout for materialized columns metadata (in seconds)
MATERIALIZED_COLUMNS_CACHE_TIMEOUT: int = get_from_env("MATERIALIZED_COLUMNS_CACHE_TIMEOUT", 900, type_cast=int)
diff --git a/products/metrics/backend/facade/api.py b/products/metrics/backend/facade/api.py
index 068c85e87edd..2a85abb4a039 100644
--- a/products/metrics/backend/facade/api.py
+++ b/products/metrics/backend/facade/api.py
@@ -8,11 +8,15 @@ so import-linter's strict-mode contract holds.
import math
import datetime as dt
from collections.abc import Sequence
+from contextlib import nullcontext
from dataclasses import replace
+from functools import partial
from typing import Any
import structlog
+from posthog.schema import DateRange, MetricsDashboardVariable, MetricsDashboardVariableSource
+
from posthog.hogql import ast
from posthog.hogql.constants import HogQLGlobalSettings
from posthog.hogql.database.schema.metrics import HOGQL_MAX_BYTES_TO_READ_FOR_METRICS_USER_QUERIES
@@ -24,6 +28,7 @@ from posthog.models import Team, User
from products.error_tracking.backend.facade.api import list_spike_events
from products.metrics.backend.anomaly import characterize_anomaly as _characterize_anomaly
+from products.metrics.backend.date_range import metrics_query_date_range
from products.metrics.backend.facade.contracts import (
CompanionMetric,
DashboardImportRequest,
@@ -63,6 +68,9 @@ from products.metrics.backend.metric_names_query_runner import MetricNamesQueryR
from products.metrics.backend.metric_query_runner import MAX_RAW_SERIES
from products.metrics.backend.metric_samples_query_runner import build_metric_query_runner
from products.metrics.backend.metrics_overview_query_runner import MetricsOverviewQueryRunner
+from products.metrics.backend.query_concurrency import query_concurrency_slots
+from products.metrics.backend.variable_options import check_options_window, compile_regex, list_variable_options
+from products.metrics.backend.variables import from_dashboard, with_values
logger = structlog.get_logger(__name__)
@@ -496,6 +504,55 @@ def list_metric_attribute_values(
return runner.run()
+def check_metric_variable_regex(regex: str | None) -> None:
+ """Raises `ValueError` for a variable regex that the options cannot use."""
+ compile_regex(regex)
+
+
+def list_metric_variable_options(
+ *,
+ team: Team,
+ source: MetricsDashboardVariableSource,
+ query: str | None = None,
+ custom_values: list[str] | None = None,
+ regex: str | None = None,
+ variables: Sequence[MetricsDashboardVariable] = (),
+ date_from: str | None = None,
+ date_to: str | None = None,
+) -> list[str]:
+ """`variables` are the selections of the other variables, which the query can refer to as `$name`.
+
+ `date_from` and `date_to` are a dashboard date filter, such as `-7d`, and resolve as they do for tiles.
+ Raises `ValueError` for a bad query or window, and `ConcurrencyLimitExceeded` when the organization
+ runs too many queries.
+ """
+ # `all` looks up the earliest event, which is a query, so it counts against the same limits.
+ lookup = query_concurrency_slots(team) if date_from == "all" else nullcontext()
+ try:
+ with lookup:
+ window = metrics_query_date_range(team, DateRange(date_from=date_from, date_to=date_to))
+ resolved_from, resolved_to = window.date_from(), window.date_to()
+ except (ValueError, OverflowError) as exc:
+ raise ValueError("The window of the variable options is not a valid date range.") from exc
+ check_options_window(resolved_from, resolved_to)
+ # The caller sends the options that All stands for, so an empty list is a known empty set.
+ selections = {
+ name: with_values(variable, variable.values) if variable.is_all else variable
+ for name, variable in from_dashboard(variables).items()
+ }
+ return list_variable_options(
+ team=team,
+ source=source,
+ query=query,
+ custom_values=custom_values,
+ regex=regex,
+ variables=selections,
+ date_from=resolved_from,
+ date_to=resolved_to,
+ slot=partial(query_concurrency_slots, team),
+ )
+
+
def list_metric_event_samples(
*,
team: Team,
diff --git a/products/metrics/backend/hogql_queries/metrics_query_runner.py b/products/metrics/backend/hogql_queries/metrics_query_runner.py
index 464eeb25032e..c5a0a34871f6 100644
--- a/products/metrics/backend/hogql_queries/metrics_query_runner.py
+++ b/products/metrics/backend/hogql_queries/metrics_query_runner.py
@@ -7,12 +7,16 @@ bypasses the facade, so the viewer, MCP tools, and insights all share one
query path.
"""
+from contextlib import nullcontext
+from functools import partial
from typing import TYPE_CHECKING, Optional, cast
from posthog.schema import (
CachedMetricsQueryResponse,
DashboardFilter,
DateRange,
+ MetricsDashboardVariable,
+ MetricsDashboardVariableSource,
MetricsQuery,
MetricsQueryLanguage,
MetricsQueryPoint,
@@ -24,6 +28,7 @@ from posthog.hogql import ast
from posthog.hogql.errors import ExposedHogQLError
from posthog.api.snuffle_proxy import SNUFFLE_API_FEATURE_FLAG
+from posthog.clickhouse.query_tagging import get_query_tag_value
from posthog.hogql_queries.query_runner import AnalyticsQueryRunner
from posthog.permissions import posthog_feature_flag_enabled
from posthog.shared_link_user import SharedLinkUser
@@ -48,11 +53,16 @@ from products.metrics.backend.facade.enums import (
MetricType,
)
from products.metrics.backend.promql import run_promql_range
+from products.metrics.backend.query_concurrency import query_concurrency_slots
from products.metrics.backend.sql_mode import run_metrics_sql
+from products.metrics.backend.variable_options import fill_all_values
from products.metrics.backend.variables import (
+ Variable,
+ all_definitions,
filter_referenced_names,
from_dashboard,
from_query,
+ query_definitions,
referenced_names,
sql_referenced_names,
substitute_filters,
@@ -69,6 +79,7 @@ if TYPE_CHECKING:
class MetricsQueryRunner(AnalyticsQueryRunner[MetricsQueryResponse]):
query: MetricsQuery
cached_response: CachedMetricsQueryResponse
+ _loaded_variables: dict[str, Variable] | None = None
def validate_query_runner_access(self, user: "User") -> bool:
user_access_control = UserAccessControl(user=user, team=self.team)
@@ -83,7 +94,7 @@ class MetricsQueryRunner(AnalyticsQueryRunner[MetricsQueryResponse]):
):
raise UserAccessControlError("metrics", "viewer")
# PromQL runs through Snuffle, which has its own private-alpha flag.
- if self.query.language == MetricsQueryLanguage.PROMQL and not posthog_feature_flag_enabled(
+ if self._sends_promql() and not posthog_feature_flag_enabled(
SNUFFLE_API_FEATURE_FLAG,
str(user.distinct_id),
organization_id=self.team.organization_id,
@@ -111,7 +122,7 @@ class MetricsQueryRunner(AnalyticsQueryRunner[MetricsQueryResponse]):
):
raise UserAccessControlError("metrics", "viewer")
# A team can have metrics without Snuffle, so a PromQL insight checks the Snuffle flag on every run.
- if self.query.language == MetricsQueryLanguage.PROMQL and not posthog_feature_flag_enabled(
+ if self._sends_promql() and not posthog_feature_flag_enabled(
SNUFFLE_API_FEATURE_FLAG,
distinct_id,
organization_id=self.team.organization_id,
@@ -196,11 +207,44 @@ class MetricsQueryRunner(AnalyticsQueryRunner[MetricsQueryResponse]):
return run_promql_range(self.team, text, date_from, date_to, self.query.interval)
if not (self.query.sql or "").strip():
raise ValueError("Write a SQL query first.")
- sql = substitute_sql(self.query.sql or "", variables)
+ sql = substitute_sql(self.query.sql or "", self._load_all_options())
if sql.unresolved:
raise ValueError(unresolved_message(sql.unresolved))
return run_metrics_sql(self.team, sql.sql, date_from, date_to, self.query.interval, sql.placeholders)
+ def _call_with_rate_limits(self, *, dashboard_id: int | None) -> tuple[MetricsQueryResponse, float]:
+ # Tiles that wait for one shared load of the All options must not hold the runner's concurrency slots.
+ if get_query_tag_value("api_queries_budgeted"):
+ self._enforce_api_queries_budget()
+ try:
+ self._load_all_options(dashboard_id)
+ except ValueError as exc:
+ raise ExposedHogQLError(str(exc)) from exc
+ return super()._call_with_rate_limits(dashboard_id=dashboard_id)
+
+ def _load_all_options(self, dashboard_id: int | None = None) -> dict[str, Variable]:
+ if self._loaded_variables is None:
+ self._enforce_alpha_gates_without_a_request_user()
+ variables = from_query(self.query.variables)
+ definitions = query_definitions(self.query.variables)
+ if definitions and self.query.language == MetricsQueryLanguage.SQL:
+ slot = partial(query_concurrency_slots, self.team, dashboard_id=dashboard_id)
+ date_range = metrics_query_date_range(self.team, self.query.dateRange)
+ # `all` looks up the earliest event, which is a query, so it counts against the same limits.
+ with slot() if self.query.dateRange and self.query.dateRange.date_from == "all" else nullcontext():
+ date_from, date_to = date_range.date_from(), date_range.date_to()
+ variables = fill_all_values(
+ team=self.team,
+ names=sql_referenced_names(self.query.sql),
+ variables=variables,
+ definitions=definitions,
+ date_from=date_from,
+ date_to=date_to,
+ slot=slot,
+ )
+ self._loaded_variables = variables
+ return self._loaded_variables
+
def _calculate(self) -> MetricsQueryResponse:
self._enforce_alpha_gates_without_a_request_user()
try:
@@ -243,11 +287,31 @@ class MetricsQueryRunner(AnalyticsQueryRunner[MetricsQueryResponse]):
def _apply_variables(self, dashboard_filter: DashboardFilter) -> None:
# The values go into the query, so each selection has its own cache entry and reaches the async worker.
+ # No options query runs here: this step comes before the access checks and the cache lookup.
+ sql_names = sql_referenced_names(self.query.sql)
+ variables = from_dashboard(dashboard_filter.metricVariables)
+ definitions: dict[str, MetricsDashboardVariable] = {}
+ if (self.query.language or MetricsQueryLanguage.BUILDER) == MetricsQueryLanguage.SQL:
+ definitions = all_definitions(
+ sql_names,
+ variables,
+ {definition.name: definition for definition in dashboard_filter.metricVariables or []},
+ )
names = dict.fromkeys(
[
*referenced_names(self.query.promql),
- *sql_referenced_names(self.query.sql),
+ *sql_names,
*(name for clause in self.query.clauses for name in filter_referenced_names(clause.filters)),
+ *(name for definition in definitions.values() for name in referenced_names(definition.query)),
]
)
- self.query.variables = to_query(from_dashboard(dashboard_filter.metricVariables), names) or None
+ self.query.variables = to_query(variables, names, definitions) or None
+
+ def _sends_promql(self) -> bool:
+ """Whether the query runs PromQL on Snuffle: as its language, or to load the options of a variable set to All."""
+ if self.query.language == MetricsQueryLanguage.PROMQL:
+ return True
+ return any(
+ definition.source == MetricsDashboardVariableSource.PROMQL
+ for definition in query_definitions(self.query.variables).values()
+ )
diff --git a/products/metrics/backend/presentation/api.py b/products/metrics/backend/presentation/api.py
index 837b892e679b..99634670ab07 100644
--- a/products/metrics/backend/presentation/api.py
+++ b/products/metrics/backend/presentation/api.py
@@ -17,14 +17,20 @@ from drf_spectacular.types import OpenApiTypes
from drf_spectacular.utils import extend_schema, extend_schema_field
from rest_framework import serializers, status, viewsets
from rest_framework.decorators import action
-from rest_framework.exceptions import ParseError, PermissionDenied
+from rest_framework.exceptions import ParseError, PermissionDenied, Throttled
from rest_framework.request import Request
from rest_framework.response import Response
+from posthog.schema import MetricsDashboardVariable, MetricsDashboardVariableSelection, MetricsDashboardVariableSource
+
from posthog.api.documentation import _FallbackSerializer
+from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGE
from posthog.api.routing import TeamAndOrgViewSetMixin
-from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
+from posthog.api.snuffle_proxy import SNUFFLE_API_FEATURE_FLAG
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
+from posthog.clickhouse.query_tagging import Feature, Product, get_query_tag_value, tag_queries
from posthog.event_usage import report_user_action
+from posthog.hogql_queries.query_runner import api_queries_budget_enforcement_enabled, get_api_queries_budget_status
from posthog.models import User
from posthog.permissions import PostHogFeatureFlagPermission, posthog_feature_flag_enabled
from posthog.rate_limit import ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle
@@ -32,6 +38,7 @@ from posthog.utils import relative_date_parse
from products.metrics.backend.facade.api import (
characterize_metric_anomaly,
+ check_metric_variable_regex,
get_metrics_overview,
list_metric_attribute_keys,
list_metric_attribute_values,
@@ -39,6 +46,7 @@ from products.metrics.backend.facade.api import (
list_metric_event_samples,
list_metric_names,
list_metric_picker_names,
+ list_metric_variable_options,
run_metric_query,
team_has_metrics,
)
@@ -692,6 +700,111 @@ class _MetricAttributeValuesResponseSerializer(serializers.Serializer):
)
+class MetricVariableSource(models.TextChoices):
+ CUSTOM = "custom", "custom"
+ PROMQL = "promql", "promql"
+ SQL = "sql", "sql"
+ TEXTBOX = "textbox", "textbox"
+
+
+class _MetricVariableSelectionSerializer(serializers.Serializer):
+ name = serializers.RegexField(
+ r"^[A-Za-z_][A-Za-z0-9_]*$",
+ max_length=128,
+ help_text="Variable name. Queries refer to it as `$name`.",
+ )
+ values = serializers.ListField(
+ child=serializers.CharField(max_length=1024, allow_blank=True, trim_whitespace=False),
+ max_length=1000,
+ help_text="The selected values. When `all` is true, the options that All stands for.",
+ )
+ all = serializers.BooleanField(required=False, default=False, help_text="True when the variable is set to All.")
+ allValue = serializers.CharField(
+ required=False,
+ allow_null=True,
+ allow_blank=True,
+ default=None,
+ max_length=1024,
+ trim_whitespace=False,
+ help_text="The text that All puts in a PromQL query. `.*` when it is empty.",
+ )
+
+
+class _MetricVariableOptionsRequestSerializer(serializers.Serializer):
+ source = serializers.ChoiceField( # type: ignore[assignment]
+ choices=MetricVariableSource.choices,
+ help_text="Where the options come from: a static list, a PromQL query, a SQL query, or free text.",
+ )
+ query = serializers.CharField(
+ required=False,
+ allow_null=True,
+ allow_blank=True,
+ default=None,
+ max_length=10_000,
+ help_text=(
+ "The options query when `source` is `promql` or `sql`. It can refer to other variables as `$name`. "
+ "PromQL supports label_values(label), label_values(selector, label), label_names(), metrics(regex) "
+ "and query_result(expr). SQL returns the options in its first column."
+ ),
+ )
+ customValues = serializers.ListField(
+ child=serializers.CharField(max_length=1024, allow_blank=True, trim_whitespace=False),
+ required=False,
+ allow_null=True,
+ default=None,
+ max_length=1000,
+ help_text="The options when `source` is `custom`.",
+ )
+ regex = serializers.CharField(
+ required=False,
+ allow_null=True,
+ allow_blank=True,
+ default=None,
+ max_length=1024,
+ trim_whitespace=False,
+ help_text=(
+ "A regex that filters the options of a query, as `pattern` or `/pattern/flags`. "
+ "With a capture group, each option is its first group. An invalid regex is rejected before any query runs."
+ ),
+ )
+ variables = _MetricVariableSelectionSerializer(
+ many=True,
+ required=False,
+ default=list,
+ help_text="The current selection of the other dashboard variables that the query can refer to.",
+ )
+ dateFrom = serializers.CharField(
+ required=False,
+ allow_null=True,
+ default=None,
+ max_length=64,
+ help_text=(
+ "Start of the window that options come from, as in a dashboard date filter: "
+ "ISO 8601 or a relative date such as '-7d'. Defaults to 24 hours ago."
+ ),
+ )
+ dateTo = serializers.CharField(
+ required=False,
+ allow_null=True,
+ default=None,
+ max_length=64,
+ help_text="End of the window that options come from, in the same format as dateFrom. Defaults to now.",
+ )
+
+ def validate_regex(self, value: str | None) -> str | None:
+ try:
+ check_metric_variable_regex(value)
+ except ValueError as exc:
+ raise serializers.ValidationError(str(exc)) from exc
+ return value
+
+
+class _MetricVariableOptionsResponseSerializer(serializers.Serializer):
+ results = serializers.ListField(
+ child=serializers.CharField(), help_text="The options of the variable, at most 1,000."
+ )
+
+
class _MetricSamplesBodySerializer(serializers.Serializer):
metricName = serializers.CharField(
required=False,
@@ -1008,6 +1121,69 @@ class MetricsViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):
return Response({"results": results}, status=status.HTTP_200_OK)
+ @extend_schema(
+ request=_MetricVariableOptionsRequestSerializer,
+ responses={200: _MetricVariableOptionsResponseSerializer},
+ )
+ @action(
+ detail=False,
+ methods=["POST"],
+ required_scopes=["metrics:read"],
+ throttle_classes=[ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle],
+ )
+ def variable_options(self, request: Request, *args, **kwargs) -> Response:
+ """The options of one dashboard variable, for the variable selects on dashboards."""
+ tag_queries(product=Product.METRICS, feature=Feature.QUERY)
+
+ body = _MetricVariableOptionsRequestSerializer(data=request.data)
+ body.is_valid(raise_exception=True)
+ data = body.validated_data
+ # PromQL goes to Snuffle, which has its own gate, as the PromQL insights and the Prometheus API do.
+ if data["source"] == MetricVariableSource.PROMQL and not posthog_feature_flag_enabled(
+ SNUFFLE_API_FEATURE_FLAG,
+ str(cast(User, request.user).distinct_id),
+ organization_id=self.team.organization_id,
+ team_id=self.team.pk,
+ ):
+ raise PermissionDenied("PromQL variable queries are not enabled for this project.")
+ # API keys are budgeted as on /query, so SQL options debit the reads they make.
+ if get_query_tag_value("access_method") == "personal_api_key":
+ tag_queries(api_queries_budgeted=True)
+ if api_queries_budget_enforcement_enabled(self.team):
+ budget = get_api_queries_budget_status(self.team)
+ if budget is not None and budget.remaining_bytes <= 0:
+ raise Throttled(
+ wait=budget.retry_after_seconds,
+ detail="ClickHouse query byte budget exhausted. Try again later.",
+ )
+ variables = [
+ MetricsDashboardVariable(
+ name=selection["name"],
+ source=MetricsDashboardVariableSource.CUSTOM,
+ includeAll=selection["all"],
+ allValue=selection["allValue"],
+ current=MetricsDashboardVariableSelection(values=selection["values"], all=selection["all"]),
+ )
+ for selection in data["variables"]
+ ]
+ try:
+ results = list_metric_variable_options(
+ team=self.team,
+ source=MetricsDashboardVariableSource(data["source"]),
+ query=data["query"],
+ custom_values=data["customValues"],
+ regex=data["regex"],
+ variables=variables,
+ date_from=data["dateFrom"],
+ date_to=data["dateTo"],
+ )
+ except ConcurrencyLimitExceeded as exc:
+ raise Throttled(detail=CONCURRENCY_LIMIT_USER_MESSAGE) from exc
+ except ValueError as exc:
+ raise ParseError(str(exc))
+
+ return Response({"results": results}, status=status.HTTP_200_OK)
+
@extend_schema(request=_MetricQueryRequestSerializer, responses={200: _MetricQueryResponseSerializer})
@action(
detail=False,
diff --git a/products/metrics/backend/promql.py b/products/metrics/backend/promql.py
index 7be11f6eba80..cee6105d300b 100644
--- a/products/metrics/backend/promql.py
+++ b/products/metrics/backend/promql.py
@@ -45,24 +45,17 @@ def _error_message(response: requests.Response) -> str:
return str(payload.get("error") or f"The PromQL query failed with status {response.status_code}.")
-def run_promql_range(
- team: Team, expr: str, date_from: dt.datetime, date_to: dt.datetime, interval: str | None
-) -> list[MetricSeries]:
- resolved = _resolve_interval(date_from, date_to, interval)
- step = _interval_step(resolved)
- start = _align_to_interval(date_from, resolved, tzinfo=team.timezone_info)
+def snuffle_payload(
+ team: Team,
+ method: str,
+ path: str,
+ *,
+ data: dict[str, str] | None = None,
+ params: dict[str, list[str]] | None = None,
+) -> dict[str, Any]:
+ """Send one Prometheus API request to Snuffle and return its JSON body. Errors are safe to show."""
try:
- response = snuffle_request(
- team.pk,
- "POST",
- "/api/v1/query_range",
- data={
- "query": expr,
- "start": str(start.timestamp()),
- "end": str(date_to.timestamp()),
- "step": str(int(step.total_seconds())),
- },
- )
+ response = snuffle_request(team.pk, method, path, data=data, params=params)
except SnuffleNotConfiguredError as exc:
raise PromQLQueryError("PromQL queries are not available on this PostHog instance.") from exc
except requests.Timeout as exc:
@@ -78,8 +71,28 @@ def run_promql_range(
payload = response.json()
except ValueError as exc:
raise PromQLQueryError("The PromQL query backend sent a response that cannot be read.") from exc
- if payload.get("status") != "success":
+ if not isinstance(payload, dict) or payload.get("status") != "success":
raise PromQLQueryError(_error_message(response))
+ return payload
+
+
+def run_promql_range(
+ team: Team, expr: str, date_from: dt.datetime, date_to: dt.datetime, interval: str | None
+) -> list[MetricSeries]:
+ resolved = _resolve_interval(date_from, date_to, interval)
+ step = _interval_step(resolved)
+ start = _align_to_interval(date_from, resolved, tzinfo=team.timezone_info)
+ payload = snuffle_payload(
+ team,
+ "POST",
+ "/api/v1/query_range",
+ data={
+ "query": expr,
+ "start": str(start.timestamp()),
+ "end": str(date_to.timestamp()),
+ "step": str(int(step.total_seconds())),
+ },
+ )
data = payload.get("data") or {}
result_type = data.get("resultType")
diff --git a/products/metrics/backend/query_concurrency.py b/products/metrics/backend/query_concurrency.py
new file mode 100644
index 000000000000..aa57c12cb745
--- /dev/null
+++ b/products/metrics/backend/query_concurrency.py
@@ -0,0 +1,44 @@
+"""Concurrency slots for metrics queries that run outside a query runner."""
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+from django.conf import settings
+
+from posthog.clickhouse.client.limit import (
+ app_org_concurrency_slot,
+ get_api_team_rate_limiter,
+ get_app_dashboard_queries_rate_limiter,
+)
+from posthog.clickhouse.query_tagging import get_query_tag_value, is_api_key_access_method
+from posthog.constants import AvailableFeature
+from posthog.models import Team
+
+
+def _api_key_concurrency_limit(team: Team) -> int | None:
+ if not settings.EE_AVAILABLE or not settings.API_QUERIES_ENABLED:
+ return None
+ feature = team.organization.get_available_feature(AvailableFeature.API_QUERIES_CONCURRENCY)
+ return feature.get("limit") if feature else None
+
+
+@contextmanager
+def query_concurrency_slots(team: Team, dashboard_id: int | None = None) -> Iterator[None]:
+ """Hold the slots a query runner holds, so the query counts against the same limits.
+
+ Browser sessions skip the API throttles, so the per-organization slot is what bounds them.
+ Raises `ConcurrencyLimitExceeded` when no slot frees up in time.
+ """
+ is_api_key_access = is_api_key_access_method(get_query_tag_value("access_method"))
+ with (
+ get_api_team_rate_limiter().run(
+ is_api=is_api_key_access,
+ team_id=team.pk,
+ limit=_api_key_concurrency_limit(team) if is_api_key_access else None,
+ ),
+ app_org_concurrency_slot(team),
+ get_app_dashboard_queries_rate_limiter().run(
+ org_id=team.organization_id, dashboard_id=dashboard_id, team_id=team.pk, is_api=is_api_key_access
+ ),
+ ):
+ yield
diff --git a/products/metrics/backend/sql_mode.py b/products/metrics/backend/sql_mode.py
index 29929d25b90a..452e164a75da 100644
--- a/products/metrics/backend/sql_mode.py
+++ b/products/metrics/backend/sql_mode.py
@@ -30,7 +30,7 @@ from posthog.hogql.database.schema.metrics import (
MetricsTable,
)
from posthog.hogql.errors import ExposedHogQLError
-from posthog.hogql.parser import parse_select
+from posthog.hogql.parser import parse_expr, parse_select
from posthog.hogql.query import execute_hogql_query
from posthog.hogql.resolver import resolve_types
from posthog.hogql.visitor import TraversingVisitor
@@ -239,3 +239,74 @@ def run_metrics_sql(
],
fill=0.0 if on_interval_starts else None,
)
+
+
+_VARIABLE_VALUE_COLUMN = "__variable_value"
+
+
+def _is_by_name(query: ast.SelectQuery | ast.SelectSetQuery) -> bool:
+ if not isinstance(query, ast.SelectSetQuery):
+ return False
+ return (
+ any(node.set_operator.endswith("BY NAME") for node in query.subsequent_select_queries)
+ or _is_by_name(query.initial_select_query)
+ or any(_is_by_name(node.select_query) for node in query.subsequent_select_queries)
+ )
+
+
+def _name_first_column(query: ast.SelectQuery | ast.SelectSetQuery) -> str:
+ """The name of the first column, aliased when it has none. The other columns stay, so ORDER BY can use them."""
+ select_query = query
+ while isinstance(select_query, ast.SelectSetQuery):
+ select_query = select_query.initial_select_query
+ if not select_query.select:
+ raise ExposedHogQLError("A SQL variable query must select the values in its first column.")
+ first = select_query.select[0]
+ if isinstance(first, ast.Field) and first.chain[-1] == "*":
+ raise ExposedHogQLError("A SQL variable query must name its first column. SELECT * is not supported.")
+ if isinstance(first, ast.Alias):
+ return first.alias
+ if _is_by_name(query):
+ # The other queries of the set match the first column by its name, so it keeps that name.
+ if isinstance(first, ast.Field) and len(first.chain) == 1 and isinstance(first.chain[0], str):
+ return first.chain[0]
+ raise ExposedHogQLError("A SQL variable query with a BY NAME set must name its first column with AS.")
+ select_query.select[0] = ast.Alias(alias=_VARIABLE_VALUE_COLUMN, expr=first)
+ return _VARIABLE_VALUE_COLUMN
+
+
+def run_metrics_sql_values(
+ team: Team,
+ sql: str,
+ date_from: dt.datetime,
+ date_to: dt.datetime,
+ limit: int,
+ variables: Mapping[str, ast.Expr] | None = None,
+ max_length: int | None = None,
+) -> list[str]:
+ """ClickHouse removes duplicates, drops values longer than `max_length` and stops at the limit, so only the
+ returned values leave the database."""
+ inner = _parse_metrics_sql(team, sql, date_from, date_to, None, variables)
+ name = _name_first_column(inner)
+ query = parse_select(
+ "SELECT DISTINCT {value} FROM {inner} WHERE {short} LIMIT {limit}",
+ placeholders={
+ "value": ast.Field(chain=[name]),
+ "inner": inner,
+ "short": parse_expr(
+ "lengthUTF8(toString({value})) <= {max_length}",
+ placeholders={"value": ast.Field(chain=[name]), "max_length": ast.Constant(value=max_length)},
+ )
+ if max_length is not None
+ else ast.Constant(value=True),
+ "limit": ast.Constant(value=limit),
+ },
+ )
+ response = execute_hogql_query(
+ query_type="MetricsSQLVariableQuery",
+ query=query,
+ team=team,
+ workload=Workload.LOGS,
+ settings=_QUERY_SETTINGS,
+ )
+ return [str(row[0]) for row in response.results or [] if row and row[0] is not None]
diff --git a/products/metrics/backend/tests/test_api.py b/products/metrics/backend/tests/test_api.py
index 33ac9e49bdd5..9fff51975487 100644
--- a/products/metrics/backend/tests/test_api.py
+++ b/products/metrics/backend/tests/test_api.py
@@ -1,15 +1,28 @@
+import time
+from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
+from functools import partial
+from typing import Any
import pytest
from posthog.test.base import APIBaseTest
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
+from django.test import override_settings
from django.utils import timezone
+import requests
from parameterized import parameterized
from rest_framework import status
+from posthog.schema import DateRange, MetricsDashboardVariable, MetricsDashboardVariableSource
+
+from posthog.hogql import ast
+from posthog.hogql.errors import ExposedHogQLError
+
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.constants import AvailableFeature
+from posthog.errors import CHQueryErrorTooManyBytes
from posthog.models.organization import OrganizationMembership
from posthog.models.personal_api_key import PersonalAPIKey
from posthog.models.user import User
@@ -18,6 +31,15 @@ from posthog.models.utils import generate_random_token_personal, hash_key_value
from products.access_control.backend.facade.user_access_control import AccessControlLevelResource
from products.access_control.backend.models.access_control import AccessControl
from products.error_tracking.backend.facade.testing import create_issue, create_spike_event
+from products.metrics.backend.date_range import metrics_query_date_range
+from products.metrics.backend.variable_options import (
+ _shared_options,
+ apply_regex,
+ cache as options_cache,
+ fill_all_values,
+ list_variable_options,
+)
+from products.metrics.backend.variables import Variable, substitute_sql
class TestMetricsValuesApi(APIBaseTest):
@@ -34,6 +56,442 @@ class TestMetricsValuesApi(APIBaseTest):
assert response.json()["attr"] == "limit"
+OPTIONS_SQL = "SELECT DISTINCT service_name FROM posthog.metrics"
+
+
+@override_settings(SNUFFLE_APM_URL="http://snuffle.test:9091")
+class TestMetricsVariableOptionsApi(APIBaseTest):
+ def _post(self, body: dict[str, Any]) -> Any:
+ return self.client.post(f"/api/projects/{self.team.id}/metrics/variable_options/", body, format="json")
+
+ def _fill_all(self, date_from: Any, date_to: Any) -> list[str]:
+ filled = fill_all_values(
+ team=self.team,
+ names=["job"],
+ variables={"job": Variable(values=(), is_all=True, all_value=None, options=())},
+ definitions={
+ "job": MetricsDashboardVariable(
+ name="job", source=MetricsDashboardVariableSource.SQL, query=OPTIONS_SQL, includeAll=True
+ )
+ },
+ date_from=date_from,
+ date_to=date_to,
+ )
+ return list(filled["job"].values)
+
+ def test_tiles_that_miss_the_cache_at_once_run_the_options_query_once(self) -> None:
+ def slow_options(*args: Any, **kwargs: Any) -> list[str]:
+ time.sleep(0.3)
+ return ["api", "worker"]
+
+ date_to = timezone.now()
+ with patch("products.metrics.backend.variable_options.run_metrics_sql_values", side_effect=slow_options) as run:
+ with ThreadPoolExecutor(max_workers=8) as executor:
+ results = list(executor.map(lambda _: self._fill_all(date_to - timedelta(hours=6), date_to), range(8)))
+
+ assert run.call_count == 1
+ assert results == [["api", "worker"]] * 8
+
+ def test_the_dashboard_select_fills_the_cache_that_tiles_read(self) -> None:
+ window = metrics_query_date_range(self.team, DateRange(date_from="-6h"))
+ too_long = "x" * 1025
+ with patch(
+ "products.metrics.backend.variable_options.run_metrics_sql_values",
+ return_value=["api", "worker", too_long],
+ ) as run:
+ response = self._post({"source": "sql", "query": OPTIONS_SQL, "dateFrom": "-6h"})
+ assert response.status_code == status.HTTP_200_OK
+ assert response.json() == {"results": ["api", "worker"]}
+ assert self._fill_all(window.date_from(), window.date_to()) == ["api", "worker"]
+
+ assert run.call_count == 1
+
+ @parameterized.expand(
+ [
+ ("wider_than_tiles", {"dateFrom": "-60d"}),
+ ("start_after_end", {"dateFrom": "-1h", "dateTo": "-2h"}),
+ ]
+ )
+ def test_rejects_windows_that_tiles_reject(self, _name: str, window: dict[str, str]) -> None:
+ with patch("products.metrics.backend.variable_options.run_metrics_sql_values") as run:
+ response = self._post({"source": "sql", "query": OPTIONS_SQL, **window})
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ run.assert_not_called()
+
+ def test_only_option_queries_that_run_take_a_concurrency_slot(self) -> None:
+ with patch("products.metrics.backend.variable_options.run_metrics_sql_values", return_value=["api"]) as run:
+ assert self._post({"source": "sql", "query": OPTIONS_SQL}).status_code == status.HTTP_200_OK
+ with patch(
+ "products.metrics.backend.query_concurrency.app_org_concurrency_slot",
+ side_effect=ConcurrencyLimitExceeded("full"),
+ ):
+ cached = self._post({"source": "sql", "query": OPTIONS_SQL})
+ throttled = self._post({"source": "sql", "query": f"{OPTIONS_SQL} WHERE 1 = 1"})
+
+ assert cached.status_code == status.HTTP_200_OK
+ assert throttled.status_code == status.HTTP_429_TOO_MANY_REQUESTS
+ assert run.call_count == 1
+
+ def test_a_failed_options_query_is_shared_with_the_next_caller(self) -> None:
+ with patch(
+ "products.metrics.backend.variable_options.run_metrics_sql_values",
+ side_effect=ExposedHogQLError("Unknown column"),
+ ) as run:
+ responses = [self._post({"source": "sql", "query": OPTIONS_SQL}) for _ in range(2)]
+
+ assert [response.status_code for response in responses] == [status.HTTP_400_BAD_REQUEST] * 2
+ assert all("Unknown column" in response.json()["detail"] for response in responses)
+ assert run.call_count == 1
+
+ def test_all_of_a_list_ignores_a_regex_left_over_from_a_query(self) -> None:
+ filled = fill_all_values(
+ team=self.team,
+ names=["env"],
+ variables={"env": Variable(values=(), is_all=True, all_value=None, options=())},
+ definitions={
+ "env": MetricsDashboardVariable(
+ name="env",
+ source=MetricsDashboardVariableSource.CUSTOM,
+ customValues=["api", "web"],
+ regex="^api",
+ includeAll=True,
+ )
+ },
+ date_from=timezone.now() - timedelta(hours=1),
+ date_to=timezone.now(),
+ )
+
+ assert filled["env"].values == ("api", "web")
+
+ @parameterized.expand(
+ [
+ (
+ "chain_without_a_loop",
+ {"v1": "$v0", "v2": "$v1", "v3": "$v2", "v4": "$v3", "v5": "$v4", "v6": "$v5"},
+ None,
+ ),
+ ("loop", {"v1": "$v2", "v2": "$v1"}, "loop"),
+ ]
+ )
+ def test_all_expansion_reports_only_real_loops(
+ self, _name: str, parents: dict[str, str], error: str | None
+ ) -> None:
+ all_value = Variable(values=(), is_all=True, all_value=None, options=())
+ definitions = {
+ "v0": MetricsDashboardVariable(
+ name="v0", source=MetricsDashboardVariableSource.CUSTOM, customValues=["a"], includeAll=True
+ ),
+ **{
+ name: MetricsDashboardVariable(
+ name=name,
+ source=MetricsDashboardVariableSource.SQL,
+ query=f"SELECT service_name FROM posthog.metrics WHERE service_name IN {parent}",
+ includeAll=True,
+ )
+ for name, parent in parents.items()
+ },
+ }
+ names = [name for name in definitions if name != "v0"]
+ with patch("products.metrics.backend.variable_options.run_metrics_sql_values", return_value=["a"]):
+ fill = partial(
+ fill_all_values,
+ team=self.team,
+ names=names[-1:],
+ variables=dict.fromkeys(definitions, all_value),
+ definitions=definitions,
+ date_from=timezone.now() - timedelta(hours=1),
+ date_to=timezone.now(),
+ )
+ if error:
+ with pytest.raises(ValueError, match=error):
+ fill()
+ else:
+ assert fill()[names[-1]].values == ("a",)
+
+ def test_all_expansion_loads_each_variable_once(self) -> None:
+ all_value = Variable(values=(), is_all=True, all_value=None, options=())
+ definitions = {
+ f"v{index}": MetricsDashboardVariable(
+ name=f"v{index}",
+ source=MetricsDashboardVariableSource.SQL,
+ query=" UNION ALL ".join(
+ ["SELECT 'a'", *(f"SELECT 'a' WHERE 'a' IN $v{parent}" for parent in reversed(range(index)))]
+ ),
+ includeAll=True,
+ )
+ for index in range(8)
+ }
+ with (
+ patch("products.metrics.backend.variable_options.run_metrics_sql_values", return_value=["a"]),
+ patch(
+ "products.metrics.backend.variable_options.list_variable_options", wraps=list_variable_options
+ ) as load,
+ ):
+ fill_all_values(
+ team=self.team,
+ names=["v7"],
+ variables=dict.fromkeys(definitions, all_value),
+ definitions=definitions,
+ date_from=timezone.now() - timedelta(hours=1),
+ date_to=timezone.now(),
+ )
+
+ assert load.call_count == 8
+
+ def test_all_expansion_stops_before_its_values_outgrow_a_query(self) -> None:
+ definitions = {
+ name: MetricsDashboardVariable(
+ name=name, source=MetricsDashboardVariableSource.SQL, query="SELECT 1", includeAll=True
+ )
+ for name in ("a", "b")
+ }
+ all_value = Variable(values=(), is_all=True, all_value=None, options=())
+ with (
+ patch("products.metrics.backend.variable_options.run_metrics_sql_values", return_value=["x" * 6]),
+ patch("products.metrics.backend.variable_options.MAX_QUERY_SIZE_BYTES", 30),
+ ):
+ with pytest.raises(ValueError, match="too many values"):
+ fill_all_values(
+ team=self.team,
+ names=["a", "b"],
+ variables=dict.fromkeys(definitions, all_value),
+ definitions=definitions,
+ date_from=timezone.now() - timedelta(hours=1),
+ date_to=timezone.now(),
+ )
+
+ @parameterized.expand(
+ [
+ ("chain_longer_than_the_cap", 60, timedelta(hours=1), "at most 50 variables"),
+ ("window_wider_than_tiles", 1, timedelta(days=60), "at most 31 days"),
+ ]
+ )
+ def test_all_expansion_limits(self, _name: str, length: int, window: timedelta, error: str) -> None:
+ all_value = Variable(values=(), is_all=True, all_value=None, options=())
+ definitions = {
+ f"v{index}": MetricsDashboardVariable(
+ name=f"v{index}",
+ source=MetricsDashboardVariableSource.SQL,
+ query=f"SELECT 'a' WHERE 'a' IN $v{index - 1}" if index else "SELECT 'a'",
+ includeAll=True,
+ )
+ for index in range(length)
+ }
+ with patch("products.metrics.backend.variable_options.run_metrics_sql_values", return_value=["a"]) as run:
+ with pytest.raises(ValueError, match=error):
+ fill_all_values(
+ team=self.team,
+ names=[f"v{length - 1}"],
+ variables=dict.fromkeys(definitions, all_value),
+ definitions=definitions,
+ date_from=timezone.now() - window,
+ date_to=timezone.now(),
+ )
+
+ run.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("promql_with_a_key", {"source": "promql", "query": "query_result(up)"}, True, 429),
+ ("sql_with_a_key", {"source": "sql", "query": OPTIONS_SQL}, True, 429),
+ ("list_in_the_browser", {"source": "custom", "customValues": ["a"]}, False, 200),
+ ]
+ )
+ def test_options_respect_the_query_byte_budget(
+ self, _name: str, body: dict[str, Any], with_key: bool, expected_status: int
+ ) -> None:
+ exhausted = MagicMock(remaining_bytes=0, retry_after_seconds=30)
+ headers = {}
+ if with_key:
+ token = generate_random_token_personal()
+ PersonalAPIKey.objects.create(
+ label="options", user=self.user, secure_value=hash_key_value(token), scopes=["metrics:read"]
+ )
+ self.client.logout()
+ headers = {"authorization": f"Bearer {token}"}
+ with (
+ patch(
+ "products.metrics.backend.presentation.api.api_queries_budget_enforcement_enabled", return_value=True
+ ),
+ patch("products.metrics.backend.presentation.api.get_api_queries_budget_status", return_value=exhausted),
+ patch("posthog.api.snuffle_proxy.internal_requests.request") as request,
+ patch("products.metrics.backend.variable_options.run_metrics_sql_values") as run,
+ ):
+ response = self.client.post(
+ f"/api/projects/{self.team.id}/metrics/variable_options/", body, format="json", headers=headers
+ )
+
+ assert response.status_code == expected_status
+ request.assert_not_called()
+ run.assert_not_called()
+
+ def test_windows_in_the_same_minutes_run_on_the_same_bounds(self) -> None:
+ start = timezone.now().replace(second=0, microsecond=0) - timedelta(hours=1)
+ with patch("products.metrics.backend.variable_options.run_metrics_sql_values", return_value=["a"]) as run:
+ for seconds in (10, 50):
+ window = {"dateFrom": (start + timedelta(seconds=seconds)).isoformat()}
+ self._post({"source": "sql", "query": OPTIONS_SQL, **window})
+
+ assert run.call_count == 1
+ assert run.call_args.args[2] == start
+
+ def test_sql_options_drop_long_values_in_clickhouse(self) -> None:
+ with patch("products.metrics.backend.variable_options.run_metrics_sql_values", return_value=["a"]) as run:
+ self._post({"source": "sql", "query": OPTIONS_SQL})
+ self._post({"source": "sql", "query": OPTIONS_SQL, "regex": "a(.)"})
+
+ assert [call.kwargs["max_length"] for call in run.call_args_list] == [1024, 16 * 1024]
+
+ def test_a_selection_set_to_all_without_options_is_an_empty_set(self) -> None:
+ with patch("products.metrics.backend.variable_options.run_metrics_sql_values", return_value=["a"]) as run:
+ response = self._post(
+ {
+ "source": "sql",
+ "query": f"{OPTIONS_SQL} WHERE service_name IN $zone",
+ "variables": [{"name": "zone", "values": [], "all": True}],
+ }
+ )
+
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ assert run.call_args.args[5] == {"metric_variable_zone": ast.Tuple(exprs=[])}
+
+ def test_all_without_options_is_an_empty_set(self) -> None:
+ with patch("products.metrics.backend.variable_options.run_metrics_sql_values", return_value=[]):
+ filled = fill_all_values(
+ team=self.team,
+ names=["job"],
+ variables={"job": Variable(values=(), is_all=True, all_value=None, options=())},
+ definitions={
+ "job": MetricsDashboardVariable(
+ name="job", source=MetricsDashboardVariableSource.SQL, query=OPTIONS_SQL, includeAll=True
+ )
+ },
+ date_from=timezone.now() - timedelta(hours=1),
+ date_to=timezone.now(),
+ )
+
+ substitution = substitute_sql("SELECT 1 WHERE service_name IN $job", filled)
+ assert substitution.placeholders == {"metric_variable_job": ast.Tuple(exprs=[])}
+
+ def test_the_regex_applies_before_the_option_length_limit(self) -> None:
+ line = 'up{instance="api", ' + "x" * 1100 + "}"
+ with patch("products.metrics.backend.variable_options.run_metrics_sql_values", return_value=[line, "web"]):
+ response = self._post({"source": "sql", "query": OPTIONS_SQL, "regex": '/instance="([^"]+)"/'})
+
+ assert response.json() == {"results": ["api"]}
+
+ def test_a_regex_group_that_does_not_match_gives_no_option(self) -> None:
+ assert apply_regex(["web", "api-web"], "^(api-)?web$") == ["api-"]
+
+ def test_a_load_that_outlives_its_lock_keeps_the_new_lock(self) -> None:
+ def slow_load() -> list[str]:
+ options_cache.set("options:lock", "taken-over", 90)
+ return ["a"]
+
+ assert _shared_options("options", slow_load) == ["a"]
+ assert options_cache.get("options:lock") == "taken-over"
+
+ def test_clickhouse_limits_are_shown_to_the_user(self) -> None:
+ with patch(
+ "products.metrics.backend.variable_options.run_metrics_sql_values",
+ side_effect=CHQueryErrorTooManyBytes("Limit for rows or bytes to read exceeded"),
+ ):
+ response = self._post({"source": "sql", "query": OPTIONS_SQL})
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert "bytes to read" in response.json()["detail"]
+
+ def test_custom_options_drop_duplicates_and_blanks(self) -> None:
+ response = self._post({"source": "custom", "customValues": ["b", "a", "b", ""]})
+ assert response.status_code == status.HTTP_200_OK
+ assert response.json() == {"results": ["b", "a"]}
+
+ @parameterized.expand(
+ [
+ (
+ "vector",
+ {"resultType": "vector", "result": [{"metric": {"job": "api"}, "value": [1, "2"]}]},
+ ['{job="api"} 2 1000'],
+ ),
+ ("scalar", {"resultType": "scalar", "result": [1, "2"]}, ["2"]),
+ ]
+ )
+ def test_query_result_reads_each_result_type(self, _name: str, data: dict[str, Any], expected: list[str]) -> None:
+ upstream = MagicMock(spec=requests.Response, status_code=200, headers={})
+ upstream.json.return_value = {"status": "success", "data": data}
+ with patch("posthog.api.snuffle_proxy.internal_requests.request", return_value=upstream):
+ response = self._post({"source": "promql", "query": "query_result(1 + 1)"})
+
+ assert response.json() == {"results": expected}
+
+ def test_label_values_resolves_the_parent_variable(self) -> None:
+ upstream = MagicMock(spec=requests.Response, status_code=200, headers={})
+ upstream.json.return_value = {"status": "success", "data": ["i-1", "i-2"]}
+ with patch("posthog.api.snuffle_proxy.internal_requests.request", return_value=upstream) as request:
+ response = self._post(
+ {
+ "source": "promql",
+ "query": 'label_values(up{job=~"$job"}, instance)',
+ "variables": [{"name": "job", "values": ["api", "web"]}],
+ }
+ )
+
+ assert response.status_code == status.HTTP_200_OK
+ assert response.json() == {"results": ["i-1", "i-2"]}
+ method, url = request.call_args.args
+ assert (method, url) == ("GET", "http://snuffle.test:9091/api/v1/label/instance/values")
+ assert request.call_args.kwargs["params"]["match[]"] == ['up{job=~"(api|web)"}']
+
+ @parameterized.expand(
+ [
+ ("label_values", "label_values(up, job)", "/api/v1/label/job/values", ["up"]),
+ ("metrics_regex", "metrics(^http_)", "/api/v1/label/__name__/values", ['{__name__=~".*(?:^http_).*"}']),
+ ]
+ )
+ def test_promql_queries_match_and_limit_on_snuffle(
+ self, _name: str, query: str, path: str, match: list[str]
+ ) -> None:
+ upstream = MagicMock(spec=requests.Response, status_code=200, headers={})
+ upstream.json.return_value = {"status": "success", "data": ["a"]}
+ with patch("posthog.api.snuffle_proxy.internal_requests.request", return_value=upstream) as request:
+ response = self._post({"source": "promql", "query": query})
+
+ assert response.status_code == status.HTTP_200_OK
+ assert request.call_args.args[1] == f"http://snuffle.test:9091{path}"
+ params = request.call_args.kwargs["params"]
+ assert (params["match[]"], params["limit"]) == (match, ["1000"])
+
+ def test_promql_queries_need_the_snuffle_flag(self) -> None:
+ def flag_enabled(flag: str, *args: Any, **kwargs: Any) -> bool:
+ return flag != "logs-metrics-snuffle-api"
+
+ with (
+ patch("posthoganalytics.feature_enabled", side_effect=flag_enabled),
+ patch("posthog.api.snuffle_proxy.internal_requests.request") as request,
+ ):
+ response = self._post({"source": "promql", "query": "query_result(up)"})
+
+ assert response.status_code == status.HTTP_403_FORBIDDEN
+ request.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("parent_without_value", {"source": "promql", "query": 'label_values(up{job="$job"}, instance)'}, "$job"),
+ ("unknown_function", {"source": "promql", "query": "sum(up)"}, "label_values"),
+ ("bad_label", {"source": "promql", "query": "label_values(up, 1abc)"}, "not a label name"),
+ ("empty_query", {"source": "sql", "query": " "}, "Write a query"),
+ ("invalid_regex", {"source": "promql", "query": "label_values(job)", "regex": "("}, "regex cannot be used"),
+ ]
+ )
+ def test_bad_queries_are_rejected_with_400(self, _name: str, body: dict, message: str) -> None:
+ with patch("posthog.api.snuffle_proxy.internal_requests.request") as request:
+ response = self._post(body)
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert message in response.json()["detail"]
+ request.assert_not_called()
+
+
class TestMetricsErrorSpikesApi(APIBaseTest):
def test_returns_spikes_from_error_tracking_in_the_window(self) -> None:
issue_id = create_issue(team_id=self.team.id, name="Boom")
@@ -165,6 +623,7 @@ class TestMetricsAccessControl(APIBaseTest):
("samples", "POST", {}),
("error_spikes", "GET", {}),
("characterize", "POST", {}),
+ ("variable_options", "POST", {}),
]
)
def test_none_access_blocks_every_metrics_action_before_validation(
diff --git a/products/metrics/backend/tests/test_metrics_query_languages.py b/products/metrics/backend/tests/test_metrics_query_languages.py
index 5f659649ee96..7eec63e62b6c 100644
--- a/products/metrics/backend/tests/test_metrics_query_languages.py
+++ b/products/metrics/backend/tests/test_metrics_query_languages.py
@@ -35,6 +35,7 @@ from products.access_control.backend.facade.user_access_control import UserAcces
from products.metrics.backend.facade.contracts import MetricPoint
from products.metrics.backend.hogql_queries.metrics_query_runner import MetricsQueryRunner
from products.metrics.backend.series import rank_and_fill_series
+from products.metrics.backend.sql_mode import run_metrics_sql_values
from products.metrics.backend.tests._seeder import seed_metric
FIXTURES: dict[str, dict[str, Any]] = json.loads(
@@ -134,6 +135,11 @@ class TestMetricsSqlMode(ClickhouseTestMixin, APIBaseTest):
@parameterized.expand(
[
("one_value", MetricsDashboardVariableSelection(values=["api"]), {"api"}),
+ (
+ "all_loads_the_values_of_the_variable",
+ MetricsDashboardVariableSelection(values=[], all=True),
+ {"api", "worker"},
+ ),
("injected_value_is_a_constant", MetricsDashboardVariableSelection(values=["api' OR 1=1 --"]), set()),
]
)
@@ -166,6 +172,31 @@ class TestMetricsSqlMode(ClickhouseTestMixin, APIBaseTest):
assert {series.labels["service_name"] for series in runner.calculate().results} == expected
+ @parameterized.expand(
+ [
+ (
+ "order_columns",
+ "SELECT service_name, count() AS n FROM posthog.metrics GROUP BY service_name ORDER BY n DESC",
+ ),
+ (
+ "by_name_union",
+ "SELECT service_name FROM posthog.metrics WHERE service_name = 'api' "
+ "UNION ALL BY NAME SELECT service_name FROM posthog.metrics WHERE service_name = 'worker'",
+ ),
+ ]
+ )
+ def test_sql_variable_options_keep_the_names_of_their_columns(self, _name: str, sql: str) -> None:
+ self._seed()
+ options = run_metrics_sql_values(
+ self.team,
+ sql,
+ NOW - dt.timedelta(hours=2),
+ NOW,
+ 1000,
+ max_length=3,
+ )
+ assert sorted(options) == ["api"]
+
def test_generated_sql_matches_the_builder_engine(self) -> None:
self._seed()
for name, fixture in FIXTURES.items():
@@ -459,6 +490,47 @@ class TestMetricsPromQLMode(APIBaseTest):
self._runner('up{job="$job"}').calculate()
request.assert_not_called()
+ def _sql_runner_with_all(self, source: str, query: str) -> MetricsQueryRunner:
+ runner = MetricsQueryRunner(
+ query=MetricsQuery(
+ clauses=[],
+ language="sql",
+ sql="SELECT now() AS time, 1 AS value WHERE 'a' IN $job",
+ dateRange=DATE_RANGE,
+ ),
+ team=self.team,
+ )
+ runner.apply_dashboard_filters(
+ DashboardFilter(
+ metricVariables=[
+ MetricsDashboardVariable(
+ name="job",
+ source=source,
+ query=query,
+ includeAll=True,
+ current=MetricsDashboardVariableSelection(values=[], all=True),
+ )
+ ]
+ )
+ )
+ return runner
+
+ def test_sql_tile_loads_promql_all_options_only_after_the_snuffle_gate(self) -> None:
+ def flag_enabled(flag: str, *args: Any, **kwargs: Any) -> bool:
+ return flag != "logs-metrics-snuffle-api"
+
+ with patch("posthog.api.snuffle_proxy.internal_requests.request") as request:
+ runner = self._sql_runner_with_all("promql", "label_values(up, job)")
+ request.assert_not_called()
+ with patch("posthoganalytics.feature_enabled", side_effect=flag_enabled):
+ with pytest.raises(UserAccessControlError):
+ runner.validate_query_runner_access(self.user)
+
+ def test_all_options_errors_are_shown_to_the_user(self) -> None:
+ runner = self._sql_runner_with_all("sql", "SELECT FROM")
+ with pytest.raises(ExposedHogQLError):
+ runner.calculate()
+
def test_empty_scalar_result_has_no_series(self) -> None:
payload = {"status": "success", "data": {"resultType": "scalar", "result": []}}
with patch("posthog.api.snuffle_proxy.internal_requests.request", return_value=_snuffle_response(200, payload)):
diff --git a/products/metrics/backend/tests/test_metrics_query_runner.py b/products/metrics/backend/tests/test_metrics_query_runner.py
index b4968a901b00..25a6bd770124 100644
--- a/products/metrics/backend/tests/test_metrics_query_runner.py
+++ b/products/metrics/backend/tests/test_metrics_query_runner.py
@@ -1,5 +1,7 @@
import datetime as dt
-from typing import cast
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import Any, cast
import pytest
import time_machine
@@ -24,6 +26,7 @@ from posthog.schema import (
)
from posthog.constants import AvailableFeature
+from posthog.hogql_queries.query_runner import ExecutionMode
from posthog.models import User
from posthog.models.personal_api_key import PersonalAPIKey
from posthog.models.sharing_configuration import SharingConfiguration
@@ -95,6 +98,50 @@ class TestMetricsQueryRunner(ClickhouseTestMixin, APIBaseTest):
assert runner.query.variables is not None and runner.query.variables["job"].values == ["api"]
+ def test_sql_tiles_check_the_budget_and_load_all_options_before_taking_a_concurrency_slot(self) -> None:
+ events: list[str] = []
+
+ @contextmanager
+ def slot(*args: Any, **kwargs: Any) -> Iterator[None]:
+ events.append("slot")
+ yield
+
+ def fill(*args: Any, variables: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
+ events.append("options")
+ return variables
+
+ runner = MetricsQueryRunner(
+ query=MetricsQuery(clauses=[], language="sql", sql="SELECT now() AS time, 1 AS value WHERE 'a' IN $job"),
+ team=self.team,
+ )
+ runner.apply_dashboard_filters(
+ DashboardFilter(
+ metricVariables=[
+ MetricsDashboardVariable(
+ name="job",
+ source="sql",
+ query="SELECT 'a'",
+ includeAll=True,
+ current=MetricsDashboardVariableSelection(values=[], all=True),
+ )
+ ]
+ )
+ )
+ with (
+ patch("products.metrics.backend.hogql_queries.metrics_query_runner.get_query_tag_value", return_value=True),
+ patch.object(
+ MetricsQueryRunner, "_enforce_api_queries_budget", side_effect=lambda: events.append("budget")
+ ),
+ patch("posthog.hogql_queries.query_runner.app_org_concurrency_slot", side_effect=slot),
+ patch("products.metrics.backend.hogql_queries.metrics_query_runner.fill_all_values", side_effect=fill),
+ patch("products.metrics.backend.hogql_queries.metrics_query_runner.substitute_sql") as substitute,
+ patch("products.metrics.backend.hogql_queries.metrics_query_runner.run_metrics_sql", return_value=[]),
+ ):
+ substitute.return_value.unresolved = ()
+ runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)
+
+ assert events.index("budget") < events.index("options") < events.index("slot")
+
def test_default_date_range_is_last_24_hours(self) -> None:
query = MetricsQuery(
clauses=[MetricsQueryClause(name="a", metricName="http_requests_total", aggregation="sum")]
diff --git a/products/metrics/backend/tests/test_variables.py b/products/metrics/backend/tests/test_variables.py
index a9b4d6bb065a..92a116624700 100644
--- a/products/metrics/backend/tests/test_variables.py
+++ b/products/metrics/backend/tests/test_variables.py
@@ -12,6 +12,7 @@ from posthog.schema import (
from posthog.hogql import ast
+from products.metrics.backend.variable_options import apply_regex, re2_pattern
from products.metrics.backend.variables import (
Variable,
from_dashboard,
@@ -225,3 +226,33 @@ def test_all_needs_include_all(include_all: bool) -> None:
current=MetricsDashboardVariableSelection(values=["a"], all=True),
)
assert from_dashboard([definition])["env"].is_all is include_all
+
+
+ [
+ ("plain", "^api", "^api"),
+ ("slashes", "/api-(.*)/", "api-(.*)"),
+ ("flags_become_inline", "/API-(.*)/ig", "(?i)API-(.*)"),
+ ]
+)
+def test_re2_pattern(_name: str, regex: str, expected: str) -> None:
+ assert re2_pattern(regex) == expected
+
+
+ [
+ ("filter", "^api", ["api-1", "web-1"], ["api-1"]),
+ ("numbered_group", "/api-(.*)/", ["api-1", "web-1"], ["1"]),
+ ("named_group", "api-(?<value>.*)", ["api-1"], ["1"]),
+ ("non_capturing_group", "api-(?:1|2)", ["api-1", "api-3"], ["api-1"]),
+ ("flags", "/API-(.*)/i", ["api-1"], ["1"]),
+ ("dot_matches_a_newline", "^api.*$", ["api\nworker"], ["api\nworker"]),
+ ("empty_option_without_a_group", ".*", ["", "api"], ["", "api"]),
+ ]
+)
+def test_apply_regex(_name: str, regex: str, options: list[str], expected: list[str]) -> None:
+ assert apply_regex(options, regex) == expected
+
+
+def test_apply_regex_reads_long_options_in_one_pass() -> None:
+ assert apply_regex(['up{instance="api", ' + "x" * 10_000_000 + "}"], 'instance="([^"]+)"') == ["api"]
diff --git a/products/metrics/backend/variable_options.py b/products/metrics/backend/variable_options.py
new file mode 100644
index 000000000000..870b911a5200
--- /dev/null
+++ b/products/metrics/backend/variable_options.py
@@ -0,0 +1,441 @@
+"""The options of a metrics dashboard variable, from a static list or a PromQL or SQL query."""
+
+from __future__ import annotations
+
+import re
+import json
+import time
+import uuid
+import hashlib
+import datetime as dt
+import dataclasses
+from collections.abc import Callable, Iterable, Mapping
+from contextlib import AbstractContextManager, nullcontext
+from itertools import islice
+from typing import Any
+
+from django.core.cache import caches
+from django.utils.connection import ConnectionProxy
+
+import re2
+
+from posthog.schema import MetricsDashboardVariable, MetricsDashboardVariableSource
+
+from posthog.hogql import ast
+from posthog.hogql.errors import ExposedHogQLError
+
+from posthog.clickhouse.client.connection import MAX_QUERY_SIZE_BYTES
+from posthog.dataclasses import frozen
+from posthog.errors import ExposedCHQueryError
+from posthog.models import Team
+
+from products.metrics.backend.metric_query_runner import MAX_QUERY_SPAN
+from products.metrics.backend.promql import PromQLQueryError, snuffle_payload
+from products.metrics.backend.sql_mode import run_metrics_sql_values
+from products.metrics.backend.variables import (
+ Variable,
+ promql_string_escape,
+ sql_referenced_names,
+ substitute_promql,
+ substitute_sql,
+ unresolved_message,
+ with_values,
+)
+
+MAX_VARIABLE_OPTIONS = 1000
+MAX_EXPANDED_VARIABLES = 50
+_FUNCTION = re.compile(r"^\s*(?P<name>label_values|label_names|metrics|query_result)\s*\((?P<args>.*)\)\s*$", re.DOTALL)
+_LABEL_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_OPTIONS_CACHE_SECONDS = 60
+# Tiles of one dashboard run at the same time. One of them loads the options and holds a lock while it runs.
+# The lock outlasts the slowest query, a 60 second Snuffle or ClickHouse query, so a slow query runs only once.
+_OPTIONS_LOCK_SECONDS = 90
+_OPTIONS_POLL_SECONDS = 0.1
+# A failed load is shared with the callers that wait for it, so they do not run the same failing query again.
+_OPTIONS_ERROR_SECONDS = 10
+cache = ConnectionProxy(caches, "metrics_variable_options")
+# The selection API accepts values up to this length, so a longer option could not be selected.
+MAX_OPTION_LENGTH = 1024
+# A regex can capture a short option from a longer value, so a query with a regex keeps longer values for it.
+_MAX_REGEX_SOURCE_LENGTH = 16 * 1024
+
+
+@frozen
+class LabelValuesArgs:
+ selector: str
+ label: str
+
+
+def _label_values_args(args: str) -> LabelValuesArgs:
+ """Split `selector, label` at the last comma outside brackets and quotes."""
+ depth = 0
+ quote: str | None = None
+ split_at = -1
+ index = 0
+ while index < len(args):
+ char = args[index]
+ if quote:
+ if char == "\\":
+ index += 1
+ elif char == quote:
+ quote = None
+ elif char in "\"'`":
+ quote = char
+ elif char in "({[":
+ depth += 1
+ elif char in ")}]":
+ depth -= 1
+ elif char == "," and depth == 0:
+ split_at = index
+ index += 1
+ if split_at < 0:
+ return LabelValuesArgs(selector="", label=args.strip())
+ return LabelValuesArgs(selector=args[:split_at].strip(), label=args[split_at + 1 :].strip())
+
+
+def _strings(data: object) -> list[str]:
+ return [str(item) for item in islice(data, MAX_VARIABLE_OPTIONS)] if isinstance(data, list) else []
+
+
+def _query_result_line(result: object) -> str | None:
+ # Grafana shows each sample as `metric{label="value"} value timestamp`, and the regex reads from that text.
+ if not isinstance(result, dict):
+ return None
+ labels = {str(key): str(value) for key, value in (result.get("metric") or {}).items()}
+ name = labels.pop("__name__", "")
+ sample = result.get("value")
+ if not isinstance(sample, list) or len(sample) != 2:
+ return None
+ matchers = ", ".join(f'{key}="{value}"' for key, value in sorted(labels.items()))
+ return f"{name}{{{matchers}}} {sample[1]} {int(float(sample[0]) * 1000)}"
+
+
+def _promql_options(team: Team, query: str, date_from: dt.datetime, date_to: dt.datetime) -> list[str]:
+ match = _FUNCTION.match(query)
+ if match is None:
+ raise PromQLQueryError(
+ "Use label_values(label), label_values(selector, label), label_names(), metrics(regex) "
+ "or query_result(expr) in a PromQL variable query."
+ )
+ name, args = match.group("name"), match.group("args").strip()
+ # Snuffle stops at the limit, so a label with many values does not travel in full.
+ params: dict[str, list[str]] = {
+ "start": [str(date_from.timestamp())],
+ "end": [str(date_to.timestamp())],
+ "limit": [str(MAX_VARIABLE_OPTIONS)],
+ }
+ if name == "label_values":
+ label_args = _label_values_args(args)
+ if not _LABEL_NAME.match(label_args.label):
+ raise PromQLQueryError(f'"{label_args.label}" is not a label name.')
+ if label_args.selector:
+ params["match[]"] = [label_args.selector]
+ return _strings(
+ snuffle_payload(team, "GET", f"/api/v1/label/{label_args.label}/values", params=params).get("data")
+ )
+ if name == "label_names":
+ if args:
+ params["match[]"] = [args]
+ return _strings(snuffle_payload(team, "GET", "/api/v1/labels", params=params).get("data"))
+ if name == "metrics":
+ pattern = args.strip("\"'")
+ if pattern:
+ # Grafana matches the regex anywhere in the name. A PromQL matcher matches the whole name.
+ params["match[]"] = [f'{{__name__=~".*(?:{promql_string_escape(pattern)}).*"}}']
+ return _strings(snuffle_payload(team, "GET", "/api/v1/label/__name__/values", params=params).get("data"))
+ if not args:
+ raise PromQLQueryError("query_result needs a PromQL expression.")
+ data = snuffle_payload(
+ team, "POST", "/api/v1/query", data={"query": args, "time": params["end"][0], "limit": params["limit"][0]}
+ ).get("data")
+ results = data.get("result") if isinstance(data, dict) else None
+ # A scalar or a string result is one `[timestamp, value]` pair, and Grafana shows only its value.
+ if isinstance(data, dict) and data.get("resultType") in ("scalar", "string"):
+ return [str(results[1])] if isinstance(results, list) and len(results) == 2 else []
+ lines = (_query_result_line(result) for result in results or [])
+ return list(islice((line for line in lines if line is not None), MAX_VARIABLE_OPTIONS))
+
+
+def re2_pattern(regex: str | None) -> str:
+ """The RE2 pattern of a variable regex. Grafana writes `/pattern/flags`, and RE2 takes the flags inline."""
+ pattern = (regex or "").strip()
+ if not (pattern.startswith("/") and pattern.rfind("/") > 0):
+ return pattern
+ flags = "".join(flag for flag in "ims" if flag in pattern[pattern.rfind("/") + 1 :])
+ body = pattern[1 : pattern.rfind("/")]
+ return f"(?{flags}){body}" if flags and body else body
+
+
+def compile_regex(regex: str | None) -> Any:
+ """The compiled variable regex, or None without one. Raises `ValueError` for a pattern that RE2 rejects.
+
+ RE2 runs the pattern in linear time, so a user pattern cannot backtrack without end. A dot matches a newline,
+ as in the ClickHouse functions that SQL queries use.
+ """
+ pattern = re2_pattern(regex)
+ if not pattern:
+ return None
+ try:
+ return re2.compile(f"(?s){pattern}")
+ except re2.error as exc:
+ raise ValueError(f"The variable regex cannot be used: {exc}") from exc
+
+
+def apply_regex(options: list[str], regex: str | None) -> list[str]:
+ """Keep the options that match the variable's regex. With a capture group, the option is the first group."""
+ compiled = compile_regex(regex) if options else None
+ if compiled is None:
+ return options
+ matched: list[str] = []
+ for option in options:
+ match = compiled.search(option)
+ if match is None:
+ continue
+ # An option whose group does not take part in the match has no value, as in the dashboard selects.
+ value = (match.group(1) or "") if compiled.groups else option
+ if value or not compiled.groups:
+ matched.append(value)
+ return list(dict.fromkeys(matched))
+
+
+def check_options_window(date_from: dt.datetime, date_to: dt.datetime) -> None:
+ if date_from >= date_to:
+ raise ValueError("The start of the window must be before its end.")
+ # Tiles have the same cap, so an options query cannot scan more history than a tile.
+ if date_to - date_from > MAX_QUERY_SPAN:
+ raise ValueError(f"The window of the variable options can be at most {MAX_QUERY_SPAN.days} days.")
+
+
[email protected](frozen=False)
+class _Expansion:
+ # Nested SQL variable queries share one expansion, so each variable set to All loads once per query.
+ filled: dict[str, Variable]
+ resolved: set[str] = dataclasses.field(default_factory=set)
+ started: int = 0
+ characters: int = 0
+
+
+def fill_all_values(
+ *,
+ team: Team,
+ names: Iterable[str],
+ variables: Mapping[str, Variable],
+ definitions: Mapping[str, MetricsDashboardVariable],
+ date_from: dt.datetime,
+ date_to: dt.datetime,
+ slot: Callable[[], AbstractContextManager[None]] = nullcontext,
+) -> dict[str, Variable]:
+ """SQL cannot match everything, so each variable set to All gets the values of all its options."""
+ check_options_window(date_from, date_to)
+ expansion = _Expansion(filled=dict(variables))
+ _expand(
+ team=team,
+ names=names,
+ expansion=expansion,
+ definitions=definitions,
+ date_from=date_from,
+ date_to=date_to,
+ path=frozenset(),
+ slot=slot,
+ )
+ return expansion.filled
+
+
+def _expand(
+ *,
+ team: Team,
+ names: Iterable[str],
+ expansion: _Expansion,
+ definitions: Mapping[str, MetricsDashboardVariable],
+ date_from: dt.datetime,
+ date_to: dt.datetime,
+ path: frozenset[str],
+ slot: Callable[[], AbstractContextManager[None]],
+) -> None:
+ for name in names:
+ variable = expansion.filled.get(name)
+ definition = definitions.get(name)
+ if name in expansion.resolved or variable is None or not variable.is_all or variable.values:
+ continue
+ if definition is None:
+ continue
+ if name in path:
+ raise ValueError(f"The variable queries refer to each other in a loop through ${name}.")
+ if expansion.started >= MAX_EXPANDED_VARIABLES:
+ raise ValueError(f"A query can use at most {MAX_EXPANDED_VARIABLES} variables that are set to All.")
+ expansion.started += 1
+ options = list_variable_options(
+ team=team,
+ source=definition.source,
+ query=definition.query,
+ custom_values=definition.customValues,
+ regex=definition.regex,
+ variables=expansion.filled,
+ definitions=definitions,
+ date_from=date_from,
+ date_to=date_to,
+ path=path | {name},
+ slot=slot,
+ expansion=expansion,
+ )
+ # The values go into the query text, which ClickHouse caps at MAX_QUERY_SIZE_BYTES. Half of it is left for the query.
+ expansion.characters += sum(len(option) + 4 for option in options)
+ if expansion.characters > MAX_QUERY_SIZE_BYTES // 2:
+ raise ValueError("The variables set to All have too many values for one query. Select fewer values.")
+ expansion.filled[name] = with_values(variable, options)
+ expansion.resolved.add(name)
+
+
+def _cache_key(team: Team, parts: list[Any]) -> str:
+ payload = json.dumps(parts, sort_keys=True, default=str)
+ return f"metrics:variable_options:{team.pk}:{hashlib.sha256(payload.encode()).hexdigest()}"
+
+
+def _query_cache_key(
+ team: Team,
+ source: MetricsDashboardVariableSource,
+ text: str,
+ placeholders: Mapping[str, ast.Expr],
+ date_from: dt.datetime,
+ date_to: dt.datetime,
+ regex: str | None,
+) -> str:
+ """The key of the query that runs, with the variable values in it, so a dashboard select and a tile share it."""
+ values = {
+ key: [item.value for item in expr.exprs if isinstance(item, ast.Constant)]
+ if isinstance(expr, ast.Tuple)
+ else expr.value
+ if isinstance(expr, ast.Constant)
+ else repr(expr)
+ for key, expr in placeholders.items()
+ }
+ # UTC minutes, so that the browser and the tiles, which compute "now" a few seconds apart, share the key.
+ window = [moment.astimezone(dt.UTC).isoformat(timespec="minutes") for moment in (date_from, date_to)]
+ return _cache_key(team, [source.value, text, values, window, re2_pattern(regex)])
+
+
+def _shared_options(
+ key: str, load: Callable[[], list[str]], slot: Callable[[], AbstractContextManager[None]] = nullcontext
+) -> list[str]:
+ """The options from the shared cache. When several callers miss at once, one runs `load` and the others wait.
+
+ Only the caller that holds the lock runs `load`, inside `slot`. A cache hit or a waiter takes no
+ slot. A load that fails with an error the user can see gives the waiters the same error. Any other
+ failure releases the lock, and the next waiter loads.
+ """
+ lock_key = f"{key}:lock"
+ error_key = f"{key}:error"
+ token = uuid.uuid4().hex
+ deadline = time.monotonic() + _OPTIONS_LOCK_SECONDS
+ while True:
+ options = _published_options(key, error_key)
+ if options is not None:
+ return options
+ if cache.add(lock_key, token, _OPTIONS_LOCK_SECONDS):
+ break
+ if time.monotonic() >= deadline:
+ raise ValueError("The variable values are still loading. Try again in a moment.")
+ time.sleep(_OPTIONS_POLL_SECONDS)
+ try:
+ # Another loader can publish between the read above and taking the lock.
+ options = _published_options(key, error_key)
+ if options is not None:
+ return options
+ try:
+ with slot():
+ options = [option for option in load() if len(str(option)) <= MAX_OPTION_LENGTH]
+ except (ValueError, ExposedHogQLError, ExposedCHQueryError) as exc:
+ cache.set(error_key, str(exc), _OPTIONS_ERROR_SECONDS)
+ raise
+ cache.set(key, options, _OPTIONS_CACHE_SECONDS)
+ return options
+ finally:
+ # A load that outlived the lock must not release the lock of the loader that took it over.
+ if cache.get(lock_key) == token:
+ cache.delete(lock_key)
+
+
+def _published_options(key: str, error_key: str) -> list[str] | None:
+ if (error := cache.get(error_key)) is not None:
+ raise ValueError(error)
+ return cache.get(key)
+
+
+def list_variable_options(
+ *,
+ team: Team,
+ source: MetricsDashboardVariableSource,
+ query: str | None,
+ custom_values: list[str] | None,
+ variables: Mapping[str, Variable],
+ date_from: dt.datetime,
+ date_to: dt.datetime,
+ regex: str | None = None,
+ definitions: Mapping[str, MetricsDashboardVariable] | None = None,
+ path: frozenset[str] = frozenset(),
+ slot: Callable[[], AbstractContextManager[None]] = nullcontext,
+ expansion: _Expansion | None = None,
+) -> list[str]:
+ """The options of one variable, in the order the source gives them. Raises `ValueError` for a bad query.
+
+ `definitions` lets a SQL query use another variable that is set to All, by loading its options.
+ `slot` holds a concurrency slot while the query runs. `regex` filters the options of a query, and a list
+ ignores it, because its select shows every value.
+ """
+ # The cache key holds whole minutes, so the query runs on the same whole minutes.
+ date_from = date_from.replace(second=0, microsecond=0)
+ if date_to.second or date_to.microsecond:
+ date_to = date_to.replace(second=0, microsecond=0) + dt.timedelta(minutes=1)
+ options: list[Any]
+ if source == MetricsDashboardVariableSource.CUSTOM:
+ options = [value for value in custom_values or [] if value]
+ elif source == MetricsDashboardVariableSource.TEXTBOX:
+ options = []
+ elif not (query or "").strip():
+ raise ValueError("Write a query for the variable first.")
+ elif source == MetricsDashboardVariableSource.PROMQL:
+ substitution = substitute_promql(query or "", variables)
+ if substitution.unresolved:
+ raise ValueError(unresolved_message(substitution.unresolved))
+ key = _query_cache_key(team, source, substitution.text, {}, date_from, date_to, regex)
+ options = _shared_options(
+ key, lambda: apply_regex(_promql_options(team, substitution.text, date_from, date_to), regex), slot
+ )
+ else:
+ if definitions:
+ expansion = expansion or _Expansion(filled=dict(variables))
+ _expand(
+ team=team,
+ names=sql_referenced_names(query),
+ expansion=expansion,
+ definitions=definitions,
+ date_from=date_from,
+ date_to=date_to,
+ path=path,
+ slot=slot,
+ )
+ variables = expansion.filled
+ sql = substitute_sql(query or "", variables)
+ if sql.unresolved:
+ raise ValueError(unresolved_message(sql.unresolved))
+ key = _query_cache_key(team, source, sql.sql, sql.placeholders, date_from, date_to, regex)
+ try:
+ options = _shared_options(
+ key,
+ lambda: apply_regex(
+ run_metrics_sql_values(
+ team,
+ sql.sql,
+ date_from,
+ date_to,
+ MAX_VARIABLE_OPTIONS,
+ sql.placeholders,
+ max_length=_MAX_REGEX_SOURCE_LENGTH if regex else MAX_OPTION_LENGTH,
+ ),
+ regex,
+ ),
+ slot,
+ )
+ except (ExposedHogQLError, ExposedCHQueryError) as exc:
+ raise ValueError(str(exc)) from exc
+ return list(dict.fromkeys(str(option) for option in options))[:MAX_VARIABLE_OPTIONS]
diff --git a/products/metrics/backend/variables.py b/products/metrics/backend/variables.py
index c92433f447c6..74d1175d5b1b 100644
--- a/products/metrics/backend/variables.py
+++ b/products/metrics/backend/variables.py
@@ -10,7 +10,13 @@ import dataclasses
from collections.abc import Collection, Iterable, Iterator, Mapping
from typing import Any
-from posthog.schema import MetricsDashboardVariable, MetricsFilterOp, MetricsQueryFilter, MetricsVariableValue
+from posthog.schema import (
+ MetricsDashboardVariable,
+ MetricsDashboardVariableSource,
+ MetricsFilterOp,
+ MetricsQueryFilter,
+ MetricsVariableValue,
+)
from posthog.hogql import ast
@@ -46,6 +52,8 @@ class Variable:
options: tuple[str, ...]
# Grafana puts an imported all value into the query as it is written, whatever string holds it.
all_value_as_written: bool = False
+ # True once All has the values of every option, which can be none.
+ loaded: bool = False
@frozen
@@ -246,19 +254,55 @@ def from_query(values: Mapping[str, MetricsVariableValue] | None) -> dict[str, V
}
-def to_query(variables: Mapping[str, Variable], names: Iterable[str]) -> dict[str, MetricsVariableValue]:
- """The values to set on a metrics query: only the variables it refers to, so the others do not reach its cache key."""
+def to_query(
+ variables: Mapping[str, Variable],
+ names: Iterable[str],
+ definitions: Mapping[str, MetricsDashboardVariable] | None = None,
+) -> dict[str, MetricsVariableValue]:
+ """The values to set on a metrics query: only the variables it refers to, so the others do not reach its cache key.
+
+ `definitions` are the variables whose All options the query loads when it runs.
+ """
return {
name: MetricsVariableValue(
- values=list(variable.values), all=variable.is_all or None, allValue=variable.all_value
+ values=list(variable.values),
+ all=variable.is_all or None,
+ allValue=variable.all_value,
+ definition=(definitions or {}).get(name),
)
for name in names
if (variable := variables.get(name)) is not None
}
+def query_definitions(values: Mapping[str, MetricsVariableValue] | None) -> dict[str, MetricsDashboardVariable]:
+ return {name: value.definition for name, value in (values or {}).items() if value.definition is not None}
+
+
+def all_definitions(
+ names: Iterable[str],
+ variables: Mapping[str, Variable],
+ definitions: Mapping[str, MetricsDashboardVariable],
+) -> dict[str, MetricsDashboardVariable]:
+ """The definitions that SQL needs to load the options of All: of the named variables, and of the
+ variables that their SQL queries refer to."""
+ needed: dict[str, MetricsDashboardVariable] = {}
+ queue = list(names)
+ while queue:
+ name = queue.pop()
+ variable = variables.get(name)
+ definition = definitions.get(name)
+ if name in needed or variable is None or not variable.is_all or variable.values or definition is None:
+ continue
+ # The options query needs no current selection or label.
+ needed[name] = definition.model_copy(update={"current": None, "label": None})
+ if definition.source == MetricsDashboardVariableSource.SQL:
+ queue.extend(sql_referenced_names(definition.query))
+ return needed
+
+
def with_values(variable: Variable, values: Iterable[str]) -> Variable:
- return dataclasses.replace(variable, values=tuple(values))
+ return dataclasses.replace(variable, values=tuple(values), loaded=True)
def format_promql(variable: Variable, value_format: str | None, *, context: PromQLStringContext) -> str:
@@ -342,7 +386,7 @@ def _comment_end(sql: str, start: int) -> int | None:
def _sql_placeholder(name: str, variable: Variable) -> ast.Expr:
- if variable.is_all and not variable.values:
+ if variable.is_all and not variable.values and not variable.loaded:
raise ValueError(f"${name} is set to All, but its values are not known. Select values for ${name}.")
if len(variable.values) == 1 and not variable.is_all:
return ast.Constant(value=variable.values[0])
diff --git a/products/metrics/frontend/generated/api.schemas.ts b/products/metrics/frontend/generated/api.schemas.ts
index c77e8a134e90..b680a0be8e31 100644
--- a/products/metrics/frontend/generated/api.schemas.ts
+++ b/products/metrics/frontend/generated/api.schemas.ts
@@ -1219,6 +1219,92 @@ export interface _MetricCatalogValuesParamsApi {
names: string[]
}
+/**
+ * * `custom` - custom
+ * * `promql` - promql
+ * * `sql` - sql
+ * * `textbox` - textbox
+ */
+export type MetricVariableSourceEnumApi = (typeof MetricVariableSourceEnumApi)[keyof typeof MetricVariableSourceEnumApi]
+
+export const MetricVariableSourceEnumApi = {
+ Custom: 'custom',
+ Promql: 'promql',
+ Sql: 'sql',
+ Textbox: 'textbox',
+} as const
+
+export interface _MetricVariableSelectionApi {
+ /**
+ * Variable name. Queries refer to it as `$name`.
+ * @maxLength 128
+ * @pattern ^[A-Za-z_][A-Za-z0-9_]*$
+ */
+ name: string
+ /**
+ * The selected values. When `all` is true, the options that All stands for.
+ * @maxItems 1000
+ * @items.maxLength 1024
+ */
+ values: string[]
+ /** True when the variable is set to All. */
+ all?: boolean
+ /**
+ * The text that All puts in a PromQL query. `.*` when it is empty.
+ * @maxLength 1024
+ * @nullable
+ */
+ allValue?: string | null
+}
+
+export interface _MetricVariableOptionsRequestApi {
+ /** Where the options come from: a static list, a PromQL query, a SQL query, or free text.
+ *
+ * * `custom` - custom
+ * * `promql` - promql
+ * * `sql` - sql
+ * * `textbox` - textbox */
+ source: MetricVariableSourceEnumApi
+ /**
+ * The options query when `source` is `promql` or `sql`. It can refer to other variables as `$name`. PromQL supports label_values(label), label_values(selector, label), label_names(), metrics(regex) and query_result(expr). SQL returns the options in its first column.
+ * @maxLength 10000
+ * @nullable
+ */
+ query?: string | null
+ /**
+ * The options when `source` is `custom`.
+ * @maxItems 1000
+ * @nullable
+ * @items.maxLength 1024
+ */
+ customValues?: string[] | null
+ /**
+ * A regex that filters the options of a query, as `pattern` or `/pattern/flags`. With a capture group, each option is its first group. An invalid regex is rejected before any query runs.
+ * @maxLength 1024
+ * @nullable
+ */
+ regex?: string | null
+ /** The current selection of the other dashboard variables that the query can refer to. */
+ variables?: _MetricVariableSelectionApi[]
+ /**
+ * Start of the window that options come from, as in a dashboard date filter: ISO 8601 or a relative date such as '-7d'. Defaults to 24 hours ago.
+ * @maxLength 64
+ * @nullable
+ */
+ dateFrom?: string | null
+ /**
+ * End of the window that options come from, in the same format as dateFrom. Defaults to now.
+ * @maxLength 64
+ * @nullable
+ */
+ dateTo?: string | null
+}
+
+export interface _MetricVariableOptionsResponseApi {
+ /** The options of the variable, at most 1,000. */
+ results: string[]
+}
+
export type MetricsAttributeValuesRetrieveParams = {
/**
* Lower bound (inclusive) of the window values are suggested from. ISO 8601. Defaults to 24 hours ago.
diff --git a/products/metrics/frontend/generated/api.ts b/products/metrics/frontend/generated/api.ts
index 073f04ea6374..481d272cf6ad 100644
--- a/products/metrics/frontend/generated/api.ts
+++ b/products/metrics/frontend/generated/api.ts
@@ -38,6 +38,8 @@ import type {
_MetricQueryResponseApi,
_MetricSamplesRequestApi,
_MetricSamplesResponseApi,
+ _MetricVariableOptionsRequestApi,
+ _MetricVariableOptionsResponseApi,
_MetricsOverviewResponseApi,
} from './api.schemas'
@@ -620,3 +622,23 @@ export const metricsValuesCreate = async (
body: JSON.stringify(_metricCatalogValuesParamsApi),
})
}
+
+export const getMetricsVariableOptionsCreateUrl = (projectId: string) => {
+ return `/api/projects/${projectId}/metrics/variable_options/`
+}
+
+/**
+ * The options of one dashboard variable, for the variable selects on dashboards.
+ */
+export const metricsVariableOptionsCreate = async (
+ projectId: string,
+ _metricVariableOptionsRequestApi: _MetricVariableOptionsRequestApi,
+ options?: RequestInit
+): Promise<_MetricVariableOptionsResponseApi> => {
+ return apiMutator<_MetricVariableOptionsResponseApi>(getMetricsVariableOptionsCreateUrl(projectId), {
+ ...options,
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', ...options?.headers },
+ body: JSON.stringify(_metricVariableOptionsRequestApi),
+ })
+}
diff --git a/products/metrics/frontend/generated/api.zod.ts b/products/metrics/frontend/generated/api.zod.ts
index 2369d28bd77c..40b7a1e4dd95 100644
--- a/products/metrics/frontend/generated/api.zod.ts
+++ b/products/metrics/frontend/generated/api.zod.ts
@@ -761,3 +761,93 @@ export const MetricsValuesCreateBody = /* @__PURE__ */ zod.object({
.max(metricsValuesCreateBodyNamesMax)
.describe('Exact metric names to load as a batch. Overrides value and limit.'),
})
+
+/**
+ * The options of one dashboard variable, for the variable selects on dashboards.
+ */
+export const metricsVariableOptionsCreateBodyQueryMax = 10000
+
+export const metricsVariableOptionsCreateBodyCustomValuesItemMax = 1024
+
+export const metricsVariableOptionsCreateBodyCustomValuesMax = 1000
+
+export const metricsVariableOptionsCreateBodyRegexMax = 1024
+
+export const metricsVariableOptionsCreateBodyVariablesItemNameMax = 128
+
+export const metricsVariableOptionsCreateBodyVariablesItemNameRegExp = new RegExp('^[A-Za-z_][A-Za-z0-9_]\*$')
+export const metricsVariableOptionsCreateBodyVariablesItemValuesItemMax = 1024
+
+export const metricsVariableOptionsCreateBodyVariablesItemValuesMax = 1000
+
+export const metricsVariableOptionsCreateBodyVariablesItemAllDefault = false
+export const metricsVariableOptionsCreateBodyVariablesItemAllValueMax = 1024
+
+export const metricsVariableOptionsCreateBodyDateFromMax = 64
+
+export const metricsVariableOptionsCreateBodyDateToMax = 64
+
+export const MetricsVariableOptionsCreateBody = /* @__PURE__ */ zod.object({
+ source: zod
+ .enum(['custom', 'promql', 'sql', 'textbox'])
+ .describe('\* `custom` - custom\n\* `promql` - promql\n\* `sql` - sql\n\* `textbox` - textbox')
+ .describe(
+ 'Where the options come from: a static list, a PromQL query, a SQL query, or free text.\n\n\* `custom` - custom\n\* `promql` - promql\n\* `sql` - sql\n\* `textbox` - textbox'
+ ),
+ query: zod
+ .string()
+ .max(metricsVariableOptionsCreateBodyQueryMax)
+ .nullish()
+ .describe(
+ 'The options query when `source` is `promql` or `sql`. It can refer to other variables as `$name`. PromQL supports label_values(label), label_values(selector, label), label_names(), metrics(regex) and query_result(expr). SQL returns the options in its first column.'
+ ),
+ customValues: zod
+ .array(zod.string().max(metricsVariableOptionsCreateBodyCustomValuesItemMax))
+ .max(metricsVariableOptionsCreateBodyCustomValuesMax)
+ .nullish()
+ .describe('The options when `source` is `custom`.'),
+ regex: zod
+ .string()
+ .max(metricsVariableOptionsCreateBodyRegexMax)
+ .nullish()
+ .describe(
+ 'A regex that filters the options of a query, as `pattern` or `\/pattern\/flags`. With a capture group, each option is its first group. An invalid regex is rejected before any query runs.'
+ ),
+ variables: zod
+ .array(
+ zod.object({
+ name: zod
+ .string()
+ .max(metricsVariableOptionsCreateBodyVariablesItemNameMax)
+ .regex(metricsVariableOptionsCreateBodyVariablesItemNameRegExp)
+ .describe('Variable name. Queries refer to it as `$name`.'),
+ values: zod
+ .array(zod.string().max(metricsVariableOptionsCreateBodyVariablesItemValuesItemMax))
+ .max(metricsVariableOptionsCreateBodyVariablesItemValuesMax)
+ .describe('The selected values. When `all` is true, the options that All stands for.'),
+ all: zod
+ .boolean()
+ .default(metricsVariableOptionsCreateBodyVariablesItemAllDefault)
+ .describe('True when the variable is set to All.'),
+ allValue: zod
+ .string()
+ .max(metricsVariableOptionsCreateBodyVariablesItemAllValueMax)
+ .nullish()
+ .describe('The text that All puts in a PromQL query. `.\*` when it is empty.'),
+ })
+ )
+ .optional()
+ .describe('The current selection of the other dashboard variables that the query can refer to.'),
+ dateFrom: zod
+ .string()
+ .max(metricsVariableOptionsCreateBodyDateFromMax)
+ .nullish()
+ .describe(
+ "Start of the window that options come from, as in a dashboard date filter: ISO 8601 or a relative date such as '-7d'. Defaults to 24 hours ago."
+ ),
+ dateTo: zod
+ .string()
+ .max(metricsVariableOptionsCreateBodyDateToMax)
+ .nullish()
+ .describe('End of the window that options come from, in the same format as dateFrom. Defaults to now.'),
+})
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index ba3d7b2be8c8..9d1f01457765 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -58407,9 +58407,11 @@ export namespace Schemas {
}
export interface MetricsVariableValue {
- /** True when the variable is set to All. PromQL gets `allValue` or `.*`, builder filters drop the filter, and SQL gets `values`. */
+ /** True when the variable is set to All. PromQL gets `allValue` or `.*`, builder filters drop the filter, and SQL gets every option of the variable. */
all?: boolean | null;
allValue?: string | null;
+ /** The definition of a variable set to All, so that a SQL query can load its options. */
+ definition?: MetricsDashboardVariable | null;
values: string[];
}
@@ -67695,6 +67697,22 @@ export namespace Schemas {
Retention: 'retention',
} as const;
+ /**
+ * * `custom` - custom
+ * * `promql` - promql
+ * * `sql` - sql
+ * * `textbox` - textbox
+ */
+ export type MetricVariableSourceEnum = typeof MetricVariableSourceEnum[keyof typeof MetricVariableSourceEnum];
+
+
+ export const MetricVariableSourceEnum = {
+ Custom: 'custom',
+ Promql: 'promql',
+ Sql: 'sql',
+ Textbox: 'textbox',
+ } as const;
+
export interface MetricsCreatedDashboard {
/** The id of the dashboard. */
dashboard_id: number;
@@ -117893,6 +117911,77 @@ export namespace Schemas {
results: _MetricEventSample[];
}
+ export interface _MetricVariableSelection {
+ /**
+ * Variable name. Queries refer to it as `$name`.
+ * @maxLength 128
+ * @pattern ^[A-Za-z_][A-Za-z0-9_]*$
+ */
+ name: string;
+ /**
+ * The selected values. When `all` is true, the options that All stands for.
+ * @maxItems 1000
+ * @items.maxLength 1024
+ */
+ values: string[];
+ /** True when the variable is set to All. */
+ all?: boolean;
+ /**
+ * The text that All puts in a PromQL query. `.*` when it is empty.
+ * @maxLength 1024
+ * @nullable
+ */
+ allValue?: string | null;
+ }
+
+ export interface _MetricVariableOptionsRequest {
+ /** Where the options come from: a static list, a PromQL query, a SQL query, or free text.
+ *
+ * * `custom` - custom
+ * * `promql` - promql
+ * * `sql` - sql
+ * * `textbox` - textbox */
+ source: MetricVariableSourceEnum;
+ /**
+ * The options query when `source` is `promql` or `sql`. It can refer to other variables as `$name`. PromQL supports label_values(label), label_values(selector, label), label_names(), metrics(regex) and query_result(expr). SQL returns the options in its first column.
+ * @maxLength 10000
+ * @nullable
+ */
+ query?: string | null;
+ /**
+ * The options when `source` is `custom`.
+ * @maxItems 1000
+ * @nullable
+ * @items.maxLength 1024
+ */
+ customValues?: string[] | null;
+ /**
+ * A regex that filters the options of a query, as `pattern` or `/pattern/flags`. With a capture group, each option is its first group. An invalid regex is rejected before any query runs.
+ * @maxLength 1024
+ * @nullable
+ */
+ regex?: string | null;
+ /** The current selection of the other dashboard variables that the query can refer to. */
+ variables?: _MetricVariableSelection[];
+ /**
+ * Start of the window that options come from, as in a dashboard date filter: ISO 8601 or a relative date such as '-7d'. Defaults to 24 hours ago.
+ * @maxLength 64
+ * @nullable
+ */
+ dateFrom?: string | null;
+ /**
+ * End of the window that options come from, in the same format as dateFrom. Defaults to now.
+ * @maxLength 64
+ * @nullable
+ */
+ dateTo?: string | null;
+ }
+
+ export interface _MetricVariableOptionsResponse {
+ /** The options of the variable, at most 1,000. */
+ results: string[];
+ }
+
export interface _MetricsOverviewService {
/** Service that reported metrics inside the window. */
service_name: string;