Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

40.4 KB
Raw Blame
"""Facade for metrics.

This is the ONLY module other products (and the presentation layer) are
allowed to import. Internal modules (query runners) stay behind this seam
so import-linter's strict-mode contract holds.
"""

import math
import datetime as dt
from collections.abc import Callable, Sequence
from contextlib import AbstractContextManager, nullcontext
from dataclasses import replace
from functools import partial
from typing import Any

from django.conf import settings
from django.db import connection

import structlog

from posthog.schema import DateRange, MetricsDashboardVariable, MetricsDashboardVariableSource

from posthog.hogql import ast
from posthog.hogql.constants import HogQLGlobalSettings
from posthog.hogql.database.schema.metrics import HOGQL_MAX_BYTES_TO_READ_FOR_METRICS_USER_QUERIES
from posthog.hogql.parser import parse_select
from posthog.hogql.query import execute_hogql_query

from posthog.clickhouse.client.connection import Workload
from posthog.hogql_queries.utils.parallel import run_in_parallel_threads
from posthog.models import Team, User

from products.error_tracking.backend.facade.api import list_latest_spike_events
from products.metrics.backend.anomaly import characterize_anomaly as _characterize_anomaly
from products.metrics.backend.date_range import metrics_query_date_range
from products.metrics.backend.facade.contracts import (
    CompanionMetric,
    DashboardImportRequest,
    DashboardImportStatus,
    DashboardTemplateSummary,
    IncidentContext,
    InvestigationResult,
    MetricAnomalyReport,
    MetricErrorSpike,
    MetricEventSample,
    MetricFilter,
    MetricPoint,
    MetricQueryClause,
    MetricQueryCount,
    MetricQueryRequest,
    MetricSeries,
    MetricsOverview,
    PanelQueryCheckRequest,
    PanelQueryCheckResult,
    SuggestedDashboard,
)
from products.metrics.backend.facade.enums import (
    DashboardTemplateStatus,
    FilterOp,
    MetricAggregation,
    MetricRangeFunction,
    MetricType,
)
from products.metrics.backend.formula import evaluate, parse_formula
from products.metrics.backend.has_metrics_query_runner import team_has_metrics as _team_has_metrics
from products.metrics.backend.investigation import investigate as _investigate
from products.metrics.backend.metric_attributes_query_runner import (
    MetricAttributeKeysQueryRunner,
    MetricAttributeValuesQueryRunner,
)
from products.metrics.backend.metric_event_samples_query_runner import MetricEventSamplesQueryRunner
from products.metrics.backend.metric_names_query_runner import MetricNamesQueryRunner, cached_metric_names
from products.metrics.backend.metric_query_runner import (
    MAX_QUERY_SPAN,
    MAX_RAW_SERIES,
    MetricQueryRunner,
    _pick_interval,
)
from products.metrics.backend.metric_samples_query_runner import build_metric_query_runner
from products.metrics.backend.metrics_overview_query_runner import MetricsOverviewQueryRunner
from products.metrics.backend.promql import run_promql_range
from products.metrics.backend.query_concurrency import query_concurrency_slots
from products.metrics.backend.query_counts import (
    list_metric_query_counts as _list_metric_query_counts,
    record_metric_queries,
    record_promql_query as _record_promql_query,
)
from products.metrics.backend.variable_options import check_options_window, compile_regex, list_variable_options
from products.metrics.backend.variables import from_dashboard, with_values

logger = structlog.get_logger(__name__)

# MetricQueryRunner still speaks the legacy aggregation strings; this shrinks
# as later PRs teach the runner the remaining MetricAggregation values.
# NONE maps to no aggregation, which is not a string.
_RUNNER_AGGREGATIONS: dict[MetricAggregation, str] = {
    MetricAggregation.SUM: "sum",
    MetricAggregation.AVG: "avg",
    MetricAggregation.COUNT: "count",
    MetricAggregation.MIN: "min",
    MetricAggregation.MAX: "max",
    MetricAggregation.RATE: "rate",
    MetricAggregation.INCREASE: "increase",
}


def team_has_metrics(team: Team) -> bool:
    """Return True if the given team has ingested at least one metric."""
    return _team_has_metrics(team)


def _units_by_fingerprint(team: Team, metric_names: set[str]) -> dict[int, str]:
    """One ClickHouse lookup of the ingested UCUM unit per physical series,
    keyed by `series_fingerprint` and read from `metric_series` (the same table
    the catalog reads). Per-fingerprint rather than per-name so a metric whose
    series disagree on the unit never lends one series' unit to another.
    Returns only series with a non-empty unit; a series with no unit is simply
    absent, so callers use `.get(fingerprint)`. Kept separate from the
    per-series data query so the unit costs one small grouped scan regardless
    of how many series a query returns."""
    if not metric_names:
        return {}
    names = sorted(metric_names)
    query = parse_select(
        """
            SELECT series_fingerprint, any(unit) AS unit
            FROM posthog.metric_series
            WHERE metric_name IN {names}
            GROUP BY series_fingerprint
        """,
        placeholders={"names": ast.Tuple(exprs=[ast.Constant(value=n) for n in names])},
    )
    response = execute_hogql_query(
        query_type="MetricUnitsLookup",
        query=query,
        team=team,
        workload=Workload.LOGS,  # metrics share the logs ClickHouse workload pool for now
        # Same byte cap as the data queries: without it a high-cardinality name set
        # would make this metadata scan the most expensive read of the request.
        settings=HogQLGlobalSettings(
            max_execution_time=30,
            max_bytes_to_read=HOGQL_MAX_BYTES_TO_READ_FOR_METRICS_USER_QUERIES,
            read_overflow_mode="throw",
        ),
    )
    return {int(row[0]): str(row[1]) for row in (response.results or []) if row[1]}


def _unit_for_fingerprints(fingerprints: set[int], units_by_fingerprint: dict[int, str]) -> str | None:
    """The ingested unit shared by the physical series one output series
    aggregated. The unit only applies when every contributing series carries
    the same one. A disagreement, or a mix of unit and no unit, means the
    merge has no correct single unit, so it returns None. Series that all
    carry no unit return "", which `_output_unit` needs to tell a unitless
    counter (whose rate is still per second) apart from a disagreement."""
    units = {units_by_fingerprint.get(fp, "") for fp in fingerprints}
    if len(units) != 1:
        return None
    (unit,) = units
    return unit


def _output_unit(ingested: str | None, clause: MetricQueryClause) -> str | None:
    """The unit of the values a clause returns, which differs from the
    ingested unit when the clause transforms the values. COUNT returns a
    number of series, so it has no unit, even after a range function. RATE
    divides by the bucket length in seconds, so it returns the ingested unit
    per second, as a UCUM string the frontend formatter reads. Every other
    aggregation keeps the ingested unit."""
    if ingested is None or clause.aggregation is MetricAggregation.COUNT:
        return None
    if clause.aggregation is MetricAggregation.RATE or clause.range_function is MetricRangeFunction.RATE:
        return f"{ingested or '1'}/s"
    return ingested or None


# Hard cap on series returned per clause; the largest series (by summed
# absolute value) win so the most significant groups survive truncation.
# A clause without an aggregation is capped earlier, in the query, at the same number.
MAX_SERIES_PER_CLAUSE = MAX_RAW_SERIES


def _assemble_series(
    rows: list[dict[str, Any]],
    *,
    clause: MetricQueryClause,
    grid: list[str],
    units_by_fingerprint: dict[int, str],
    zero_fill: bool = True,
) -> list[MetricSeries]:
    """Split bucketed rows into one series per label-set, zero-filled onto
    the shared grid so every series (and later, every clause of a formula)
    has identical timestamps.

    A row with a `series_key` stays apart from rows of another key, even when
    their labels match: two physical series can share a label-set."""
    by_labels: dict[tuple[tuple[tuple[str, str], ...], int | None], dict[str, float | None]] = {}
    fingerprints_by_labels: dict[tuple[tuple[tuple[str, str], ...], int | None], set[int]] = {}
    bounds_by_labels: dict[tuple[tuple[tuple[str, str], ...], int | None], set[float]] = {}
    for row in rows:
        key = (tuple(sorted(row["labels"].items())), row.get("series_key"))
        by_labels.setdefault(key, {})[row["time"]] = row["value"]
        fingerprints_by_labels.setdefault(key, set()).update(row["series_fingerprints"])
        if "histogram_bounds" in row:
            bounds_by_labels.setdefault(key, set()).update(row["histogram_bounds"])

    # Rank and truncate on the sparse values BEFORE zero-filling, so a
    # high-cardinality group-by never materializes label_sets x grid points
    # only to throw most of them away. Zero-filled points contribute nothing
    # to the magnitude, so the ranking is identical either way.
    ranked = sorted(
        by_labels.items(), key=lambda item: (-sum(abs(v) for v in item[1].values() if v is not None), item[0])
    )
    return [
        MetricSeries(
            labels=dict(key[0]),
            points=tuple(MetricPoint(time=time, value=values.get(time, 0.0 if zero_fill else None)) for time in grid),
            metric_name=clause.metric_name,
            clause=clause.name,
            unit=_output_unit(_unit_for_fingerprints(fingerprints_by_labels[key], units_by_fingerprint), clause),
            histogram_bounds=tuple(sorted(bounds_by_labels[key])) if key in bounds_by_labels else None,
        )
        for key, values in ranked[:MAX_SERIES_PER_CLAUSE]
    ]


def _labels_by_fingerprint(team: Team, metric_name: str, fingerprints: set[int]) -> dict[int, dict[str, str]]:
    """The label set of each series, for a clause that returns series unaggregated.

    Resource attributes come first, so a datapoint attribute of the same name wins.
    """
    if not fingerprints:
        return {}
    query = parse_select(
        """
            SELECT series_fingerprint, any(resource_attributes), any(attributes)
            FROM posthog.metric_series
            WHERE metric_name = {metric_name}
              AND series_fingerprint IN {fingerprints}
            GROUP BY series_fingerprint
        """,
        placeholders={
            "metric_name": ast.Constant(value=metric_name),
            "fingerprints": ast.Tuple(exprs=[ast.Constant(value=fp) for fp in sorted(fingerprints)]),
        },
    )
    response = execute_hogql_query(
        query_type="MetricSeriesLabelsLookup",
        query=query,
        team=team,
        workload=Workload.LOGS,  # metrics share the logs ClickHouse workload pool for now
        settings=HogQLGlobalSettings(
            max_execution_time=30,
            max_bytes_to_read=HOGQL_MAX_BYTES_TO_READ_FOR_METRICS_USER_QUERIES,
            read_overflow_mode="throw",
        ),
    )
    return {
        int(row[0]): {str(key): str(value) for key, value in {**(row[1] or {}), **(row[2] or {})}.items() if value}
        for row in (response.results or [])
    }


def _attach_series_labels(team: Team, metric_name: str, rows: list[dict[str, Any]]) -> None:
    """Give each row of an unaggregated clause the labels of its series."""
    labels = _labels_by_fingerprint(team, metric_name, {fp for row in rows for fp in row["series_fingerprints"]})
    for row in rows:
        (fingerprint,) = row["series_fingerprints"]
        row["labels"] = labels.get(int(fingerprint), {})
        row["series_key"] = int(fingerprint)


def _drop_shared_labels(series: list[MetricSeries]) -> list[MetricSeries]:
    """With several series, keep only the labels that tell them apart.

    A label that has the same value on every series only repeats in each legend entry.
    """
    if len(series) < 2:
        return series
    shared = {
        key for key, value in series[0].labels.items() if all(other.labels.get(key) == value for other in series[1:])
    }
    return [replace(s, labels={k: v for k, v in s.labels.items() if k not in shared}) for s in series]


def _resolve_runner_aggregation(clause: MetricQueryClause) -> str | None:
    if clause.aggregation.is_raw:
        return None
    if clause.aggregation == MetricAggregation.QUANTILE and clause.quantile == 0.95:
        return "p95"
    if clause.aggregation == MetricAggregation.HISTOGRAM_QUANTILE:
        return "histogram_quantile"
    if clause.aggregation in _RUNNER_AGGREGATIONS:
        return _RUNNER_AGGREGATIONS[clause.aggregation]
    raise ValueError(f"aggregation {clause.aggregation.value!r} is not supported yet")


def _evaluate_formula_point(
    node: Any, per_clause_points: dict[str, tuple[MetricPoint, ...]], index: int
) -> float | None:
    """One formula grid point. A null (gap) in any input propagates as a
    gap, and a result the formula overflowed to inf/NaN becomes a gap too —
    same policy as the per-clause aggregates."""
    values: dict[str, float] = {}
    for name, pts in per_clause_points.items():
        value = pts[index].value
        if value is None:
            return None
        values[name] = value
    result = evaluate(node, values)
    return result if math.isfinite(result) else None


def _evaluate_formula(
    node: Any, series_by_clause: dict[str, list[MetricSeries]], grid: list[str]
) -> list[MetricSeries]:
    """Combine clause results point-by-point on the shared grid.

    Series are matched across clauses by exact label-set equality
    (Prometheus-style one-to-one vector matching); a clause that produced a
    single ungrouped series is broadcast to every label-set instead. A
    label-set missing from any non-broadcast clause is dropped.
    """
    broadcasts: dict[str, MetricSeries] = {}
    grouped: dict[str, dict[tuple[tuple[str, str], ...], MetricSeries]] = {}
    for name, series_list in series_by_clause.items():
        if len(series_list) == 1 and not series_list[0].labels:
            broadcasts[name] = series_list[0]
        else:
            grouped[name] = {tuple(sorted(s.labels.items())): s for s in series_list}

    if grouped:
        label_sets: set[tuple[tuple[str, str], ...]] = set.intersection(
            *(set(by_labels) for by_labels in grouped.values())
        )
    else:
        label_sets = {()}

    result: list[MetricSeries] = []
    for label_set in sorted(label_sets):
        per_clause_points: dict[str, tuple[MetricPoint, ...]] = {
            name: (grouped[name][label_set].points if name in grouped else broadcasts[name].points)
            for name in series_by_clause
        }
        points = tuple(
            MetricPoint(time=time, value=_evaluate_formula_point(node, per_clause_points, index))
            for index, time in enumerate(grid)
        )
        result.append(MetricSeries(labels=dict(label_set), points=points, metric_name=None, clause="formula"))
    return result


def _closing_connection(work: Callable[[], None]) -> None:
    try:
        work()
    finally:
        # The query reads Postgres in the worker thread, and request cleanup only closes the request thread's connection.
        connection.close()


def run_metric_query(*, team: Team, request: MetricQueryRequest) -> list[MetricSeries]:
    """Execute a metric query and return one `MetricSeries` per
    (clause, label-set) pair — a single ungrouped clause yields exactly one
    series with empty labels, so consumers never branch on single-vs-multi.

    Every series of every clause shares one bucket grid (the union of
    observed buckets, zero-filled), which is what makes cross-series and
    cross-clause math line up. Series per clause are capped at
    `MAX_SERIES_PER_CLAUSE`, keeping the largest ones.

    With `formula` set, only the formula result series are returned
    (`clause="formula"`); request the clauses separately if you need the
    inputs too. The presentation layer surfaces `ValueError` as a 400.
    """
    formula_node_checked = (
        parse_formula(request.formula, frozenset(clause.name for clause in request.clauses))
        if request.formula is not None
        else None
    )
    record_metric_queries(team, (clause.metric_name for clause in request.clauses))
    runners: list[tuple[MetricQueryClause, MetricQueryRunner, bool]] = []
    for clause in request.clauses:
        runner_aggregation = _resolve_runner_aggregation(clause)
        runner = build_metric_query_runner(
            team=team,
            metric_name=clause.metric_name,
            aggregation=runner_aggregation,
            date_from=request.date_from,
            date_to=request.date_to,
            filters=clause.filters,
            group_by=clause.group_by,
            interval=request.interval,
            quantile=clause.quantile if runner_aggregation == "histogram_quantile" else None,
            metric_type=clause.metric_type.value if clause.metric_type is not None else None,
            range_function=clause.range_function.value if clause.range_function is not None else None,
        )
        runners.append((clause, runner, runner_aggregation is None))
    rows_by_clause: dict[str, list[dict[str, Any]]] = {clause.name: [] for clause in request.clauses}
    units_by_fingerprint: dict[int, str] = {}

    def run_clause(clause: MetricQueryClause, runner: MetricQueryRunner, raw: bool) -> None:
        rows = runner.run()
        if raw:
            _attach_series_labels(team, clause.metric_name, rows)
        rows_by_clause[clause.name] = rows

    def load_units() -> None:
        units_by_fingerprint.update(_units_by_fingerprint(team, {clause.metric_name for clause in request.clauses}))

    # The clause queries and the unit lookup do not depend on each other, so a tile waits for the slowest one,
    # not for their sum. A formula combines clauses of different units, so its series carry no unit.
    work: list[Callable[[], None]] = [partial(run_clause, *item) for item in runners]
    if formula_node_checked is None:
        work.append(load_units)
    # Django does not make the test transaction visible to another thread.
    if len(work) <= 1 or settings.IN_UNIT_TESTING:
        for item in work:
            item()
    else:
        run_in_parallel_threads(
            [partial(_closing_connection, item) for item in work], thread_name_prefix="metrics_clauses"
        )

    grid = sorted({row["time"] for rows in rows_by_clause.values() for row in rows})
    if not grid:
        empty_clause = "formula" if formula_node_checked is not None else request.clauses[0].name
        metric_name = None if formula_node_checked is not None else request.clauses[0].metric_name
        return [MetricSeries(labels={}, points=(), metric_name=metric_name, clause=empty_clause)]

    series_by_clause = {
        clause.name: _assemble_series(
            rows_by_clause[clause.name],
            clause=clause,
            grid=grid,
            units_by_fingerprint=units_by_fingerprint,
            zero_fill=request.zero_fill,
        )
        for clause in request.clauses
    }

    if formula_node_checked is not None:
        return _evaluate_formula(formula_node_checked, series_by_clause, grid)

    return [
        series
        for clause in request.clauses
        for series in (
            _drop_shared_labels(series_by_clause[clause.name])
            if clause.aggregation.is_raw
            else series_by_clause[clause.name]
        )
    ]


def list_metric_query_counts(*, team: Team, days: int, limit: int) -> list[MetricQueryCount]:
    """How often queries read each metric of the team in the last `days` UTC days, the most queried first."""
    return _list_metric_query_counts(team, days=days, limit=limit)


def record_promql_query(*, team: Team, expr: str) -> None:
    """Count one query for each metric that a PromQL expression selects. A failed write never raises."""
    _record_promql_query(team, expr)


def run_promql_query(
    *, team: Team, expr: str, date_from: dt.datetime, date_to: dt.datetime, interval: str | None
) -> list[MetricSeries]:
    """Run a PromQL range query on Snuffle. The caller checks the Snuffle feature flag. `date_to` is exclusive."""
    if date_to <= date_from:
        raise ValueError("date_to must be after date_from")
    if date_to - date_from > MAX_QUERY_SPAN:
        raise ValueError(f"date range too wide; the maximum span is {MAX_QUERY_SPAN.days} days")
    with query_concurrency_slots(team):
        series = run_promql_range(team, expr, date_from, date_to, interval)
    return [
        replace(s, points=tuple(p for p in s.points if dt.datetime.fromisoformat(p.time) < date_to)) for s in series
    ]


def metric_query_concurrency_slots(team: Team) -> AbstractContextManager[None]:
    """Hold the concurrency slots of a metrics query runner around `run_metric_query`.

    Use it when you call `run_metric_query` outside a query runner, so the query counts against the same limits.
    """
    return query_concurrency_slots(team)


def pick_metric_query_interval(*, date_from: dt.datetime, date_to: dt.datetime, max_buckets: int) -> str:
    """The finest `MetricQueryRequest.interval` that keeps the bucket count at or below `max_buckets`."""
    return _pick_interval(date_from, date_to, target_bucket_count=max_buckets)


def list_metric_names(
    *,
    team: Team,
    search: str = "",
    limit: int = 100,
    services: Sequence[str] = (),
    names: Sequence[str] = (),
) -> list[dict[str, Any]]:
    """List distinct metric names for the team's picker.

    Returns a list of `{"name": str, "metric_type": str}` dicts.
    Passing `services` narrows the list to names those services reported.
    Raises `ValueError` for an out-of-range limit or too many services.

    The unsearched list is cached per team and service scope for a minute;
    searches are not.
    """
    if names:
        return MetricNamesQueryRunner(team=team, services=services, names=names, limit=len(names)).run()
    return cached_metric_names(team=team, search=search, limit=limit, services=services)


def list_metric_picker_names(
    *,
    team: Team,
    search: str = "",
    limit: int = 100,
    services: Sequence[str] = (),
) -> list[dict[str, Any]]:
    """List current metric names and types for the viewer picker, without sparklines or caching."""
    return MetricNamesQueryRunner(team=team, search=search, limit=limit, services=services).run_picker()


def get_metrics_overview(*, team: Team, lookback: dt.timedelta | None = None) -> MetricsOverview:
    """Ingestion rollup for the overview page: freshness of the newest
    datapoint plus window-scoped metric/series counts per service.

    Raises `ValueError` for a non-positive lookback.
    """
    if lookback is None:
        return MetricsOverviewQueryRunner(team=team).run()
    return MetricsOverviewQueryRunner(team=team, lookback=lookback).run()


def list_metric_attribute_keys(
    *,
    team: Team,
    metric_name: str = "",
    search: str = "",
    date_from: dt.datetime | None = None,
    date_to: dt.datetime | None = None,
    limit: int = 100,
) -> list[dict[str, Any]]:
    """List attribute keys by distinct value count, from highest to lowest.

    When a metric name is provided, only series that emitted that metric in the
    recent window supply choices. Datapoint and resource attributes are merged
    into one list (filters run with scope 'auto', so the split doesn't matter
    to callers); `service_name` is always surfaced when it matches the search.
    The window defaults to the last 24 hours. Returns `{"name": str,
    "value_count": int}` dicts. Raises `ValueError` for an out-of-range limit
    or an inverted window.
    """
    runner = MetricAttributeKeysQueryRunner(
        team=team,
        metric_name=metric_name,
        search=search,
        date_from=date_from,
        date_to=date_to,
        limit=limit,
    )
    return runner.run()


def list_metric_attribute_values(
    *,
    team: Team,
    key: str,
    metric_name: str = "",
    search: str = "",
    date_from: dt.datetime | None = None,
    date_to: dt.datetime | None = None,
    limit: int = 100,
) -> list[dict[str, Any]]:
    """List observed values for one metric attribute key, most frequent first,
    for the filter bar's value autocomplete.

    When a metric name is provided, only values seen on that metric supply
    choices. `service_name`/`service.name` read the first-class column,
    matching how filters on it execute. The window defaults to the last 24 hours. Returns
    `{"id": str, "name": str, "count": int}` dicts. Raises `ValueError` for an
    empty key, an out-of-range limit, or an inverted window.
    """
    runner = MetricAttributeValuesQueryRunner(
        team=team,
        key=key,
        metric_name=metric_name,
        search=search,
        date_from=date_from,
        date_to=date_to,
        limit=limit,
    )
    return runner.run()


def check_metric_variable_regex(regex: str | None) -> None:
    """Raises `ValueError` for a variable regex that the options cannot use."""
    compile_regex(regex)


def list_metric_variable_options(
    *,
    team: Team,
    source: MetricsDashboardVariableSource,
    query: str | None = None,
    custom_values: list[str] | None = None,
    regex: str | None = None,
    variables: Sequence[MetricsDashboardVariable] = (),
    date_from: str | None = None,
    date_to: str | None = None,
) -> list[str]:
    """`variables` are the selections of the other variables, which the query can refer to as `$name`.

    `date_from` and `date_to` are a dashboard date filter, such as `-7d`, and resolve as they do for tiles.
    Raises `ValueError` for a bad query or window, and `ConcurrencyLimitExceeded` when the organization
    runs too many queries.
    """
    # `all` looks up the earliest event, which is a query, so it counts against the same limits.
    lookup = query_concurrency_slots(team) if date_from == "all" else nullcontext()
    try:
        with lookup:
            window = metrics_query_date_range(team, DateRange(date_from=date_from, date_to=date_to))
            resolved_from, resolved_to = window.date_from(), window.date_to()
    except (ValueError, OverflowError) as exc:
        raise ValueError("The window of the variable options is not a valid date range.") from exc
    check_options_window(resolved_from, resolved_to)
    # The caller sends the options that All stands for, so an empty list is a known empty set.
    selections = {
        name: with_values(variable, variable.values) if variable.is_all else variable
        for name, variable in from_dashboard(variables).items()
    }
    return list_variable_options(
        team=team,
        source=source,
        query=query,
        custom_values=custom_values,
        regex=regex,
        variables=selections,
        date_from=resolved_from,
        date_to=resolved_to,
        slot=partial(query_concurrency_slots, team),
    )


def list_metric_event_samples(
    *,
    team: Team,
    metric_name: str | None = None,
    date_from: dt.datetime,
    date_to: dt.datetime,
    trace_id: str | None = None,
    span_id: str | None = None,
    filters: Sequence[MetricFilter] = (),
    metric_type: MetricType | None = None,
    limit: int = 100,
) -> list[MetricEventSample]:
    """List individual metric emissions (the events model), newest first.

    Each sample carries its value, attributes, and trace linkage, so the
    Samples view can render raw rows and pivot to the trace behind any one.
    Pass `trace_id` for the reverse pivot — every emission on a given trace,
    across all metric names when `metric_name` is omitted (the tracing
    product's Metrics tab); `metric_name` or `trace_id` is required. Pass
    `span_id` (with `trace_id`) to narrow to one span's emissions, so the
    result stays exact even when the trace has more emissions than `limit`.
    `filters` and `metric_type` narrow the emissions to the same series a
    `run_metric_query` call with those arguments charts, so a filtered view
    and its chart agree. Both are matched against the emission's series, so
    an emission whose series row hasn't been ingested yet drops out once
    either is set; both require `metric_name`.
    Raises `ValueError` for a missing metric name and trace id, an inverted
    window, an invalid regex filter, or an out-of-range limit; the
    presentation layer surfaces these as 400s.
    """
    runner = MetricEventSamplesQueryRunner(
        team=team,
        metric_name=metric_name,
        date_from=date_from,
        date_to=date_to,
        trace_id=trace_id,
        span_id=span_id,
        filters=filters,
        metric_type=metric_type,
        limit=limit,
    )
    return [MetricEventSample(**row) for row in runner.run()]


# PoC cap: team-wide — Error Tracking spike events carry no service attribution,
# so there is no correlation key to scope them to one metric's service yet.
_ERROR_SPIKES_LIMIT = 200


def list_metric_error_spikes(*, team: Team, date_from: dt.datetime, date_to: dt.datetime) -> list[MetricErrorSpike]:
    """List Error Tracking issue spikes detected in a time window, for the
    metrics chart's error-spike overlay.

    Team-wide: Error Tracking issues carry no service attribution today, so
    this cannot be scoped to the metric's own service yet.
    """
    events = list_latest_spike_events(
        team_id=team.id,
        date_from=date_from.isoformat(),
        date_to=date_to.isoformat(),
        limit=_ERROR_SPIKES_LIMIT,
    )
    return [
        MetricErrorSpike(
            detected_at=event.detected_at.isoformat(),
            issue_id=str(event.issue.id),
            issue_name=event.issue.name,
        )
        for event in events
    ]


def characterize_metric_anomaly(
    *,
    team: Team,
    metric_name: str,
    anomaly_from: dt.datetime,
    anomaly_to: dt.datetime,
    baseline_from: dt.datetime | None = None,
    baseline_to: dt.datetime | None = None,
    aggregation: str | None = None,
    quantile: float | None = None,
    filters: tuple[MetricFilter, ...] = (),
    candidate_keys: tuple[str, ...] | None = None,
) -> MetricAnomalyReport:
    """Characterize how a metric behaves in an anomaly window vs a baseline:
    summary statistics, change magnitude/direction, the onset bucket, and
    the label values that moved the most (drilling into up to four candidate
    keys, auto-discovered from the metric's attributes when not given).

    The baseline defaults to the window of equal length immediately before
    `anomaly_from`. `aggregation` defaults by the metric's OTel type
    (counter -> rate, gauge -> avg, histogram -> histogram_quantile 0.95).
    Raises `ValueError` for invalid windows/aggregations — the presentation
    layer surfaces these as 400s.
    """
    return _characterize_anomaly(
        team=team,
        metric_name=metric_name,
        anomaly_from=anomaly_from,
        anomaly_to=anomaly_to,
        baseline_from=baseline_from,
        baseline_to=baseline_to,
        aggregation=aggregation,
        quantile=quantile,
        filters=filters,
        candidate_keys=candidate_keys,
    )


def investigate(
    *,
    team: Team,
    metric_name: str,
    anomaly_from: dt.datetime,
    anomaly_to: dt.datetime,
    baseline_from: dt.datetime | None = None,
    baseline_to: dt.datetime | None = None,
    aggregation: str | None = None,
    quantile: float | None = None,
    filters: tuple[MetricFilter, ...] = (),
    candidate_keys: tuple[str, ...] | None = None,
    companions: tuple[CompanionMetric, ...] = (),
) -> InvestigationResult:
    """Investigate a metric symptom end to end and return one structured result.

    Builds on `characterize_metric_anomaly`: it characterizes the metric over
    the anomaly window, then characterizes each `companion` over the SAME window
    to confirm or rule it out (e.g. throughput flat -> not a traffic surge),
    classifies blast radius from the movers, implicates a service for the
    logs/traces pivot, and emits re-runnable chart specs.

    The result is the single shape the agent narrates, the in-app explorer
    renders, and the incident report serializes. Raises `ValueError` for invalid
    windows/aggregations — the presentation layer surfaces these as 400s.
    """
    return _investigate(
        team=team,
        metric_name=metric_name,
        anomaly_from=anomaly_from,
        anomaly_to=anomaly_to,
        baseline_from=baseline_from,
        baseline_to=baseline_to,
        aggregation=aggregation,
        quantile=quantile,
        filters=filters,
        candidate_keys=candidate_keys,
        companions=companions,
    )


def investigate_incident(*, team: Team, context: IncidentContext) -> InvestigationResult:
    """Investigate a fired alert's metric with no timestamp math on the caller.

    Derives the anomaly window straight from `context.fired_at` (an explicit
    UTC instant) — no parsing a fire time out of prose, no timezone guesswork —
    scopes to the implicated service, and runs the full investigation. Returns
    the same `InvestigationResult` as `investigate()`. This is the entry point
    an alert's "Investigate" action calls.
    """
    filters: tuple[MetricFilter, ...] = ()
    if context.service_name:
        filters = (MetricFilter(key="service_name", op=FilterOp.EQ, value=context.service_name),)
    return _investigate(
        team=team,
        metric_name=context.metric_name,
        anomaly_from=context.fired_at - context.lookback,
        anomaly_to=context.fired_at + context.leadout,
        filters=filters,
        companions=context.companions,
    )


def start_dashboard_import(*, team: Team, user: User, request: DashboardImportRequest) -> DashboardImportStatus:
    """Start an import of a Grafana dashboard JSON model or a dashboard screenshot as a new dashboard.

    Converts what it can at once. Starts an agent task for the rest, and then the status has an id to poll.
    Raises `DashboardImportError` with a message for the user.
    """
    from products.metrics.backend.dashboard_import.importer import (  # noqa: PLC0415 — keeps the image and task code off the facade import path
        DashboardImporter,
    )

    return DashboardImporter(team=team, user=user).start(request)


def get_dashboard_import(*, team: Team, user: User, import_id: str) -> DashboardImportStatus | None:
    """The status of the user's import. Builds the dashboard when the agent finished and nothing built it yet."""
    from products.metrics.backend.dashboard_import.importer import (  # noqa: PLC0415 — keeps the image and task code off the facade import path
        DashboardImporter,
    )

    return DashboardImporter(team=team, user=user).status(import_id)


def list_dashboard_imports(*, team: Team, user: User) -> list[DashboardImportStatus]:
    """The user's recent imports that used an agent, newest first, with the progress of the ones that run."""
    from products.metrics.backend.dashboard_import.importer import (  # noqa: PLC0415 — keeps the image and task code off the facade import path
        DashboardImporter,
    )

    return DashboardImporter(team=team, user=user).recent()


def finalize_dashboard_import(*, team_id: int, import_id: str) -> None:
    """Build the dashboard of an import whose agent task ended. Safe to call more than once."""
    from products.metrics.backend.dashboard_import.importer import (  # noqa: PLC0415 — keeps the image and task code off the facade import path
        finalize_import,
    )

    finalize_import(team_id=team_id, task_id=import_id, background=True)


def check_dashboard_import_layout(*, team_id: int, import_id: str, saved_at: dt.datetime | None = None) -> None:
    """Take the agent's latest answer of a screenshot import, and send it a picture of the dashboard or end it."""
    from products.metrics.backend.dashboard_import.importer import (  # noqa: PLC0415 — keeps the image and task code off the facade import path
        check_import_layout,
    )

    check_import_layout(team_id=team_id, task_id=import_id, saved_at=saved_at)


def check_dashboard_panel_queries(
    *, team: Team, user: User, panels: Sequence[PanelQueryCheckRequest], import_id: str | None = None
) -> list[PanelQueryCheckResult]:
    """Check panel queries the way a dashboard import checks them before it builds the dashboard."""
    from products.metrics.backend.dashboard_import.importer import (  # noqa: PLC0415 — keeps the image and task code off the facade import path
        check_panel_queries,
    )

    return check_panel_queries(team=team, user=user, panels=panels, import_id=import_id)


def list_suggested_dashboards(*, team: Team) -> list[SuggestedDashboard]:
    """The approved bank dashboards that suit the metrics of the team, best fit first.

    The first call for a team that was never analyzed starts its analysis, so suggestions appear without
    waiting for its metric names to change.
    """
    from products.metrics.backend.suggested_dashboards import (  # noqa: PLC0415 — keeps the dashboard and export code off the facade import path
        discovery,
        service,
    )
    from products.metrics.backend.temporal.schedule import (  # noqa: PLC0415 — keeps temporalio off the facade import path
        start_team_analysis,
    )

    discovery.record_opened(team.id)
    if discovery.needs_analysis_on_open(team.id):
        try:
            start_team_analysis(team.id, force=True)
        except Exception:
            discovery.forget_analysis_on_open(team.id)
            logger.warning("metrics_suggested_dashboards_first_analysis_not_started", team_id=team.id)
    return service.list_suggestions(team)


def create_suggested_dashboard(*, team: Team, user: User, suggestion_id: str) -> int:
    """Create the dashboard of a suggestion as the user, and return its id. A second call returns the same id.

    Raises `SuggestedDashboardError` with a message for the user.
    """
    from products.metrics.backend.suggested_dashboards import (  # noqa: PLC0415 — keeps the dashboard and export code off the facade import path
        service,
    )

    return service.create_from_suggestion(team, user, suggestion_id)


def analyze_team_metrics_now(*, team: Team) -> None:
    """Start an analysis of the team's metric names now, even when nothing changed. For the staff review."""
    from products.metrics.backend.temporal.schedule import (  # noqa: PLC0415 — keeps temporalio off the facade import path
        start_team_analysis,
    )

    start_team_analysis(team.id, force=True)


def list_dashboard_templates(*, status: DashboardTemplateStatus | None = None) -> list[DashboardTemplateSummary]:
    """The templates of the metrics dashboard bank, newest first. Instance-wide, for the staff review."""
    from products.metrics.backend.suggested_dashboards import (  # noqa: PLC0415 — keeps the dashboard and export code off the facade import path
        service,
    )

    return service.list_templates(status)


def get_dashboard_template(*, template_id: str) -> DashboardTemplateSummary:
    from products.metrics.backend.suggested_dashboards import (  # noqa: PLC0415 — keeps the dashboard and export code off the facade import path
        service,
    )

    return service.get_template(template_id)


def open_dashboard_template_preview(*, team: Team, user: User, template_id: str) -> int:
    """The id of an unlisted dashboard in the team that shows the template with live data. Builds it when needed."""
    from products.metrics.backend.suggested_dashboards import (  # noqa: PLC0415 — keeps the dashboard and export code off the facade import path
        service,
    )

    return service.open_preview(team, user, template_id)


def approve_dashboard_template(*, user: User, template_id: str) -> DashboardTemplateSummary:
    """Approve a template. The changes on its preview dashboard become the template first."""
    from products.metrics.backend.suggested_dashboards import (  # noqa: PLC0415 — keeps the dashboard and export code off the facade import path
        service,
    )

    return service.approve(user, template_id)


def reject_dashboard_template(*, user: User, template_id: str) -> DashboardTemplateSummary:
    from products.metrics.backend.suggested_dashboards import (  # noqa: PLC0415 — keeps the dashboard and export code off the facade import path
        service,
    )

    return service.reject(user, template_id)


def regenerate_dashboard_template(*, user: User, template_id: str) -> DashboardTemplateSummary:
    from products.metrics.backend.suggested_dashboards import (  # noqa: PLC0415 — keeps the dashboard and export code off the facade import path
        service,
    )

    return service.regenerate(user, template_id)


def dashboard_template_picture(*, template_id: str, round_number: int) -> bytes | None:
    """The PNG that a generation round rendered of the preview dashboard."""
    from products.metrics.backend.suggested_dashboards import (  # noqa: PLC0415 — keeps the dashboard and export code off the facade import path
        service,
    )

    return service.picture(template_id, round_number)