Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

38.4 KB
Raw Blame
import datetime
import hashlib
import json
from abc import ABC
from collections.abc import Callable, Generator
from typing import Any

import numpy as np
import pytest

from fixtures import types
from fixtures.time import parse_timestamp

_REDUCED_METRICS_TABLES_TO_TRUNCATE = [
    "time_series_v4_reduced",
    "samples_v4_reduced_last_60s",
    "samples_v4_reduced_sum_60s",
    "time_series_v4_buffer",
    "samples_v4_buffer",
]


class MetricsTimeSeries(ABC):
    """Represents a row in the time_series_v4 table."""

    env: str
    temporality: str
    metric_name: str
    description: str
    unit: str
    type: str
    is_monotonic: bool
    fingerprint: np.uint64
    unix_milli: np.int64
    labels: str
    attrs: dict[str, str]
    scope_attrs: dict[str, str]
    resource_attrs: dict[str, str]

    def __init__(
        self,
        metric_name: str,
        labels: dict[str, str],
        timestamp: datetime.datetime,
        temporality: str = "Unspecified",
        description: str = "",
        unit: str = "",
        type_: str = "Sum",
        is_monotonic: bool = True,
        env: str = "default",
        resource_attrs: dict[str, str] = {},
        scope_attrs: dict[str, str] = {},
    ) -> None:
        # Create a copy of labels to avoid mutating the caller's dictionary
        labels = dict(labels)
        # Add metric_name to the labels to support promql queries
        labels["__name__"] = metric_name
        self.env = env
        self.metric_name = metric_name
        self.temporality = temporality
        self.description = description
        self.unit = unit
        self.type = type_
        self.is_monotonic = is_monotonic
        self.labels = json.dumps(labels, separators=(",", ":"))
        self.attrs = labels
        self.scope_attrs = scope_attrs
        self.resource_attrs = resource_attrs
        self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))

        # Calculate fingerprint from metric_name + labels
        fingerprint_str = metric_name + self.labels
        self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))

    def to_row(self) -> list:
        return [
            self.env,
            self.temporality,
            self.metric_name,
            self.description,
            self.unit,
            self.type,
            self.is_monotonic,
            self.fingerprint,
            self.unix_milli,
            self.labels,
            self.attrs,
            self.scope_attrs,
            self.resource_attrs,
        ]


class MetricsSample(ABC):
    """Represents a row in the samples_v4 table."""

    env: str
    temporality: str
    metric_name: str
    fingerprint: np.uint64
    unix_milli: np.int64
    value: np.float64
    flags: np.uint32

    def __init__(
        self,
        metric_name: str,
        fingerprint: np.uint64,
        timestamp: datetime.datetime,
        value: float,
        temporality: str = "Unspecified",
        env: str = "default",
        flags: int = 0,
    ) -> None:
        self.env = env
        self.temporality = temporality
        self.metric_name = metric_name
        self.fingerprint = fingerprint
        self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
        self.value = np.float64(value)
        self.flags = np.uint32(flags)

    def to_row(self) -> list:
        return [
            self.env,
            self.temporality,
            self.metric_name,
            self.fingerprint,
            self.unix_milli,
            self.value,
            self.flags,
        ]


class MetricsExpHist(ABC):
    """Represents a row in the exp_hist table for exponential histograms.

    `observations` must be non-empty; ClickHouse folds it into the `sketch`
    AggregateFunction state on insert.

    TODO: take an ExponentialHistogramDataPoint and compute the sketch bytes the
    way the collector does, so the fixture exercises the real write path instead
    of having ClickHouse build the state.
    """

    env: str
    temporality: str
    metric_name: str
    fingerprint: np.uint64
    unix_milli: np.int64
    count: np.uint64
    sum: np.float64
    min: np.float64
    max: np.float64
    observations: list[int]
    flags: np.uint32

    def __init__(
        self,
        metric_name: str,
        fingerprint: np.uint64,
        timestamp: datetime.datetime,
        observations: list[int],
        temporality: str = "Unspecified",
        env: str = "default",
        flags: int = 0,
    ) -> None:
        self.env = env
        self.temporality = temporality
        self.metric_name = metric_name
        self.fingerprint = fingerprint
        self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
        self.observations = observations
        self.count = np.uint64(len(observations))
        self.sum = np.float64(sum(observations))
        self.min = np.float64(min(observations))
        self.max = np.float64(max(observations))
        self.flags = np.uint32(flags)


class MetricsMetadata(ABC):
    """Represents a row in the metadata table for metric metadata."""

    temporality: str
    metric_name: str
    description: str
    unit: str
    type: str
    is_monotonic: bool
    attr_name: str
    attr_type: str
    attr_datatype: str
    attr_string_value: str
    first_reported_unix_milli: np.int64
    last_reported_unix_milli: np.int64

    def __init__(
        self,
        metric_name: str,
        attr_name: str,
        attr_type: str,
        attr_datatype: str,
        attr_string_value: str,
        timestamp: datetime.datetime,
        temporality: str = "Unspecified",
        description: str = "",
        unit: str = "",
        type_: str = "Sum",
        is_monotonic: bool = True,
    ) -> None:
        self.temporality = temporality
        self.metric_name = metric_name
        self.description = description
        self.unit = unit
        self.type = type_
        self.is_monotonic = is_monotonic
        self.attr_name = attr_name
        self.attr_type = attr_type
        self.attr_datatype = attr_datatype
        self.attr_string_value = attr_string_value
        unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
        self.first_reported_unix_milli = unix_milli
        self.last_reported_unix_milli = unix_milli

    def to_row(self) -> list:
        return [
            self.temporality,
            self.metric_name,
            self.description,
            self.unit,
            self.type,
            self.is_monotonic,
            self.attr_name,
            self.attr_type,
            self.attr_datatype,
            self.attr_string_value,
            self.first_reported_unix_milli,
            self.last_reported_unix_milli,
        ]


class Metrics(ABC):
    """High-level metric representation. Produces both time series and sample entries."""

    metric_name: str
    labels: dict[str, str]
    temporality: str
    timestamp: datetime.datetime
    value: float
    flags: int

    @property
    def time_series(self) -> MetricsTimeSeries:
        return self._time_series

    @property
    def sample(self) -> MetricsSample:
        return self._sample

    def __init__(
        self,
        metric_name: str,
        labels: dict[str, str] = {},
        timestamp: datetime.datetime | None = None,
        value: float = 0.0,
        temporality: str = "Unspecified",
        flags: int = 0,
        description: str = "",
        unit: str = "",
        type_: str = "Sum",
        is_monotonic: bool = True,
        env: str = "default",
        resource_attributes: dict[str, str] = {},
        scope_attributes: dict[str, str] = {},
    ) -> None:
        if timestamp is None:
            timestamp = datetime.datetime.now()
        self.metric_name = metric_name
        self.labels = labels
        self.temporality = temporality
        self.timestamp = timestamp
        self.value = value
        self.flags = flags

        self._time_series = MetricsTimeSeries(
            metric_name=metric_name,
            labels=labels,
            timestamp=timestamp,
            temporality=temporality,
            description=description,
            unit=unit,
            type_=type_,
            is_monotonic=is_monotonic,
            env=env,
            resource_attrs=resource_attributes,
            scope_attrs=scope_attributes,
        )

        self._sample = MetricsSample(
            metric_name=metric_name,
            fingerprint=self._time_series.fingerprint,
            timestamp=timestamp,
            value=value,
            temporality=temporality,
            env=env,
            flags=flags,
        )

    def to_dict(self) -> dict:
        return {
            "metric_name": self.metric_name,
            "labels": self.labels,
            "timestamp": self.timestamp.isoformat(),
            "value": self.value,
            "temporality": self.temporality,
            "type_": self._time_series.type,
            "is_monotonic": self._time_series.is_monotonic,
            "flags": self.flags,
            "description": self._time_series.description,
            "unit": self._time_series.unit,
            "env": self._time_series.env,
            "resource_attrs": self._time_series.resource_attrs,
            "scope_attrs": self._time_series.scope_attrs,
        }

    @classmethod
    def from_dict(
        cls,
        data: dict,
        # base_time: Optional[datetime.datetime] = None,
        metric_name_override: str | None = None,
    ) -> "Metrics":
        """
        Create a Metrics instance from a dict.

        Args:
            data: The dict containing metric data
            base_time: If provided, timestamps are shifted relative to this time.
                       The earliest timestamp in the data becomes base_time.
            metric_name_override: If provided, overrides the metric_name from data
        """
        # parse timestamp from iso format
        timestamp = parse_timestamp(data["timestamp"])

        return cls(
            metric_name=metric_name_override or data["metric_name"],
            labels=data.get("labels", {}),
            timestamp=timestamp,
            value=data["value"],
            temporality=data.get("temporality", "Unspecified"),
            flags=data.get("flags", 0),
            description=data.get("description", ""),
            unit=data.get("unit", ""),
            type_=data.get("type_", "Sum"),
            is_monotonic=data.get("is_monotonic", True),
            env=data.get("env", "default"),
            resource_attributes=data.get("resource_attrs", {}),
            scope_attributes=data.get("scope_attrs", {}),
        )

    @classmethod
    def load_from_file(
        cls,
        file_path: str,
        base_time: datetime.datetime | None = None,
        metric_name_override: str | None = None,
        label_substitutions: dict[str, str] | None = None,
    ) -> list["Metrics"]:
        """
        Load metrics from a JSONL file.

        Each line should be a JSON object representing a metric.

        Args:
            file_path: Path to the JSONL file
            base_time: If provided, all timestamps are shifted so the earliest
                       timestamp in the file maps to base_time
            metric_name_override: If provided, overrides metric_name for all metrics
            label_substitutions: If provided, any label whose value equals a key is
                                 rewritten to that key's value (placeholder substitution,
                                 e.g. {"__START_TIME__": start_time.isoformat()})
        """
        data_list = []
        with open(file_path, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                data = json.loads(line)
                if label_substitutions:
                    labels = data.get("labels", {})
                    for key, value in labels.items():
                        if value in label_substitutions:
                            labels[key] = label_substitutions[value]
                data_list.append(data)

        if not data_list:
            return []

        # If base_time provided, calculate time offset
        time_offset = datetime.timedelta(0)
        if base_time is not None:
            # Find earliest timestamp
            earliest = None
            for data in data_list:
                ts = parse_timestamp(data["timestamp"])
                if earliest is None or ts < earliest:
                    earliest = ts
            if earliest is not None:
                time_offset = base_time - earliest

        metrics = []
        for data in data_list:
            original_ts = parse_timestamp(data["timestamp"])
            adjusted_ts = original_ts + time_offset
            data["timestamp"] = adjusted_ts.isoformat()
            metrics.append(cls.from_dict(data, metric_name_override=metric_name_override))

        return metrics


class ExpHistogramMetrics(ABC):
    """High-level exponential histogram representation. Produces both time series
    and exp_hist entries."""

    metric_name: str
    labels: dict[str, str]
    temporality: str
    timestamp: datetime.datetime
    observations: list[int]

    @property
    def time_series(self) -> MetricsTimeSeries:
        return self._time_series

    @property
    def exp_hist(self) -> MetricsExpHist:
        return self._exp_hist

    def __init__(
        self,
        metric_name: str,
        observations: list[int],
        labels: dict[str, str] = {},
        timestamp: datetime.datetime | None = None,
        temporality: str = "Delta",
        flags: int = 0,
        description: str = "",
        unit: str = "",
        env: str = "default",
        resource_attributes: dict[str, str] = {},
        scope_attributes: dict[str, str] = {},
    ) -> None:
        if timestamp is None:
            timestamp = datetime.datetime.now()
        self.metric_name = metric_name
        self.labels = labels
        self.temporality = temporality
        self.timestamp = timestamp
        self.observations = observations

        self._time_series = MetricsTimeSeries(
            metric_name=metric_name,
            labels=labels,
            timestamp=timestamp,
            temporality=temporality,
            description=description,
            unit=unit,
            # the querier resolves the metric type from this column, and only an
            # ExponentialHistogram here routes the query to the sketch read
            type_="ExponentialHistogram",
            is_monotonic=False,
            env=env,
            resource_attrs=resource_attributes,
            scope_attrs=scope_attributes,
        )

        self._exp_hist = MetricsExpHist(
            metric_name=metric_name,
            fingerprint=self._time_series.fingerprint,
            timestamp=timestamp,
            observations=observations,
            temporality=temporality,
            env=env,
            flags=flags,
        )


class MetricsReducedTimeSeries(ABC):
    """Represents a row in the time_series_v4_reduced table i.e what
    the time_series_v4_reduced_mv materializes for a metric under a
    reduction rule. One row per kept-label group. `fingerprint` holds the
    reduced fingerprint and `labels` contains only the kept labels.

    The fingerprint recipe (md5, like MetricsTimeSeries) does not match the
    collector's real hash; it only needs to be consistent with the
    reduced_fingerprint used in the reduced samples rows.
    """

    def __init__(  # pylint: disable=too-many-arguments
        self,
        metric_name: str,
        kept_labels: dict[str, str],
        timestamp: datetime.datetime,
        temporality: str = "Unspecified",
        description: str = "",
        unit: str = "",
        type_: str = "Gauge",
        is_monotonic: bool = False,
        env: str = "default",
    ) -> None:
        kept_labels = dict(kept_labels)
        kept_labels["__name__"] = metric_name
        self.env = env
        # mirror time_series_v4_reduced_mv: monotonic cumulative counters are
        # reduced as deltas
        if temporality == "Cumulative" and is_monotonic:
            temporality = "Delta"
        self.temporality = temporality
        self.metric_name = metric_name
        self.description = description
        self.unit = unit
        self.type = type_
        self.is_monotonic = is_monotonic
        self.labels = json.dumps(kept_labels, separators=(",", ":"))
        self.attrs = kept_labels
        self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3) // 3600000 * 3600000)

        fingerprint_str = metric_name + self.labels
        self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))

    def to_row(self) -> list:
        return [
            self.env,
            self.temporality,
            self.metric_name,
            self.description,
            self.unit,
            self.type,
            self.is_monotonic,
            self.fingerprint,
            self.unix_milli,
            self.labels,
            self.attrs,
            {},
            {},
        ]


class MetricsReducedSampleLast60s(ABC):
    """Represents a row in the samples_v4_reduced_last_60s table. One 60s
    bucket per reduced group, as the samples_v4_reduced_last_60s_mv refresh
    would emit it (gauges and non-monotonic cumulative sums)."""

    def __init__(  # pylint: disable=too-many-arguments
        self,
        metric_name: str,
        reduced_fingerprint: np.uint64,
        timestamp: datetime.datetime,
        sum_last: float,
        min_value: float,
        max_value: float,
        sum_values: float,
        count_series: int,
        count_samples: int,
        temporality: str = "Unspecified",
        env: str = "default",
        computed_at: datetime.datetime | None = None,
    ) -> None:
        self.env = env
        self.temporality = temporality
        self.metric_name = metric_name
        self.reduced_fingerprint = reduced_fingerprint
        # buckets are 60s-aligned: intDiv(unix_milli, 60000) * 60000
        self.unix_milli = np.int64((int(timestamp.timestamp() * 1e3) // 60000) * 60000)
        self.sum_last = np.float64(sum_last)
        self.min = np.float64(min_value)
        self.max = np.float64(max_value)
        self.sum_values = np.float64(sum_values)
        self.count_series = np.uint64(count_series)
        self.count_samples = np.uint64(count_samples)
        # the refresh stamps now(); default to shortly after the bucket closes
        if computed_at is None:
            computed_at = datetime.datetime.fromtimestamp(int(self.unix_milli) / 1e3, tz=datetime.UTC) + datetime.timedelta(seconds=180)
        self.computed_at = computed_at

    def to_row(self) -> list:
        return [
            self.env,
            self.temporality,
            self.metric_name,
            self.reduced_fingerprint,
            self.unix_milli,
            self.sum_last,
            self.min,
            self.max,
            self.sum_values,
            self.count_series,
            self.count_samples,
            self.computed_at,
        ]


class MetricsReducedSampleSum60s(ABC):
    """Represents a row in the samples_v4_reduced_sum_60s table. One 60s
    bucket per reduced group for delta counters and histograms."""

    def __init__(  # pylint: disable=too-many-arguments
        self,
        metric_name: str,
        reduced_fingerprint: np.uint64,
        timestamp: datetime.datetime,
        sum_value: float,
        count_series: int,
        count_samples: int,
        temporality: str = "Delta",
        env: str = "default",
        computed_at: datetime.datetime | None = None,
    ) -> None:
        self.env = env
        self.temporality = temporality
        self.metric_name = metric_name
        self.reduced_fingerprint = reduced_fingerprint
        self.unix_milli = np.int64((int(timestamp.timestamp() * 1e3) // 60000) * 60000)
        self.sum = np.float64(sum_value)
        self.count_series = np.uint64(count_series)
        self.count_samples = np.uint64(count_samples)
        if computed_at is None:
            computed_at = datetime.datetime.fromtimestamp(int(self.unix_milli) / 1e3, tz=datetime.UTC) + datetime.timedelta(seconds=180)
        self.computed_at = computed_at

    def to_row(self) -> list:
        return [
            self.env,
            self.temporality,
            self.metric_name,
            self.reduced_fingerprint,
            self.unix_milli,
            self.sum,
            self.count_series,
            self.count_samples,
            self.computed_at,
        ]


class MetricsBufferTimeSeries(ABC):
    """Represents a row in the time_series_v4_buffer table. This is the collector's
    universal landing target under cardinality control. For a ruled metric the
    collector writes two rows per series: the raw one (is_reduced=false, full
    labels, reduced_fingerprint pointing at its group) and the group's reduced
    one (is_reduced=true, kept labels, fingerprint = reduced fingerprint)."""

    def __init__(  # pylint: disable=too-many-arguments
        self,
        metric_name: str,
        labels: dict[str, str],
        timestamp: datetime.datetime,
        reduced_fingerprint: np.uint64 | int = 0,
        is_reduced: bool = False,
        temporality: str = "Unspecified",
        description: str = "",
        unit: str = "",
        type_: str = "Gauge",
        is_monotonic: bool = False,
        env: str = "default",
    ) -> None:
        labels = dict(labels)
        labels["__name__"] = metric_name
        self.env = env
        self.temporality = temporality
        self.metric_name = metric_name
        self.description = description
        self.unit = unit
        self.type = type_
        self.is_monotonic = is_monotonic
        self.reduced_fingerprint = np.uint64(reduced_fingerprint)
        self.is_reduced = is_reduced
        self.labels = json.dumps(labels, separators=(",", ":"))
        self.attrs = labels
        self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3) // 3600000 * 3600000)

        fingerprint_str = metric_name + self.labels
        self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))

    def to_row(self) -> list:
        return [
            self.env,
            self.temporality,
            self.metric_name,
            self.description,
            self.unit,
            self.type,
            self.is_monotonic,
            self.fingerprint,
            self.reduced_fingerprint,
            self.is_reduced,
            self.unix_milli,
            self.labels,
            self.attrs,
            {},
            {},
        ]


class MetricsBufferSample(ABC):
    """Represents a row in the samples_v4_buffer table. Ruled samples carry
    the raw fingerprint plus the group's reduced_fingerprint; unruled samples
    have reduced_fingerprint = 0."""

    def __init__(  # pylint: disable=too-many-arguments
        self,
        metric_name: str,
        fingerprint: np.uint64,
        timestamp: datetime.datetime,
        value: float,
        reduced_fingerprint: np.uint64 | int = 0,
        is_monotonic: bool = False,
        temporality: str = "Unspecified",
        env: str = "default",
        flags: int = 0,
    ) -> None:
        self.env = env
        self.temporality = temporality
        self.metric_name = metric_name
        self.fingerprint = fingerprint
        self.reduced_fingerprint = np.uint64(reduced_fingerprint)
        self.is_monotonic = is_monotonic
        self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
        self.value = np.float64(value)
        self.flags = np.uint32(flags)

    def to_row(self) -> list:
        return [
            self.env,
            self.temporality,
            self.metric_name,
            self.fingerprint,
            self.reduced_fingerprint,
            self.is_monotonic,
            self.unix_milli,
            self.value,
            self.flags,
        ]


def insert_time_series_to_clickhouse(conn, time_series: list[MetricsTimeSeries]) -> None:
    """
    Insert one distributed_time_series_v4 registration row per (series, hour
    bucket), unix_milli floored to the hour — the exporter's exact shape.
    Readers floor lookup windows to these buckets: skipping per-bucket
    re-registration or keeping raw mid-hour timestamps hides series in ways
    production never sees.
    """
    time_series_map: dict[tuple[int, int], MetricsTimeSeries] = {}
    for ts in time_series:
        fp = int(ts.fingerprint)
        hour_bucket = int(ts.unix_milli) // 3_600_000
        if (fp, hour_bucket) not in time_series_map:
            ts.unix_milli = np.int64(hour_bucket * 3_600_000)
            time_series_map[(fp, hour_bucket)] = ts

    if len(time_series_map) == 0:
        return

    conn.insert(
        database="signoz_metrics",
        table="distributed_time_series_v4",
        column_names=[
            "env",
            "temporality",
            "metric_name",
            "description",
            "unit",
            "type",
            "is_monotonic",
            "fingerprint",
            "unix_milli",
            "labels",
            "attrs",
            "scope_attrs",
            "resource_attrs",
        ],
        data=[ts.to_row() for ts in time_series_map.values()],
    )


def insert_metrics_to_clickhouse(conn, metrics: list[Metrics]) -> None:
    """
    Insert metrics into ClickHouse tables.
    Handles insertion into:
    - distributed_time_series_v4 (time series metadata)
    - distributed_samples_v4 (actual sample values)
    - distributed_metadata (metric attribute metadata)

    Pure function so the seeder container can reuse the exact insert path
    used by the pytest fixture. `conn` is a clickhouse-connect Client.
    """
    insert_time_series_to_clickhouse(conn, [metric.time_series for metric in metrics])

    samples = [metric.sample for metric in metrics]
    if len(samples) > 0:
        conn.insert(
            database="signoz_metrics",
            table="distributed_samples_v4",
            column_names=[
                "env",
                "temporality",
                "metric_name",
                "fingerprint",
                "unix_milli",
                "value",
                "flags",
            ],
            data=[sample.to_row() for sample in samples],
        )

    insert_metrics_metadata_to_clickhouse(conn, metrics)


def insert_metrics_metadata_to_clickhouse(conn, metrics: list) -> None:
    """
    Insert the distributed_metadata rows describing each metric's point, resource
    and scope attributes. Accepts anything exposing `time_series`, `labels` and
    `timestamp`.
    """
    # (metric_name, attr_type, attr_name, attr_value) -> MetricsMetadata
    metadata_map: dict[tuple, MetricsMetadata] = {}
    for metric in metrics:
        ts = metric.time_series
        for attr_name, attr_value in metric.labels.items():
            key = (ts.metric_name, "point", attr_name, str(attr_value))
            if key not in metadata_map:
                metadata_map[key] = MetricsMetadata(
                    metric_name=ts.metric_name,
                    attr_name=attr_name,
                    attr_type="point",
                    attr_datatype="String",
                    attr_string_value=str(attr_value),
                    timestamp=metric.timestamp,
                    temporality=ts.temporality,
                    description=ts.description,
                    unit=ts.unit,
                    type_=ts.type,
                    is_monotonic=ts.is_monotonic,
                )
        for attr_name, attr_value in ts.resource_attrs.items():
            key = (ts.metric_name, "resource", attr_name, str(attr_value))
            if key not in metadata_map:
                metadata_map[key] = MetricsMetadata(
                    metric_name=ts.metric_name,
                    attr_name=attr_name,
                    attr_type="resource",
                    attr_datatype="String",
                    attr_string_value=str(attr_value),
                    timestamp=metric.timestamp,
                    temporality=ts.temporality,
                    description=ts.description,
                    unit=ts.unit,
                    type_=ts.type,
                    is_monotonic=ts.is_monotonic,
                )
        for attr_name, attr_value in ts.scope_attrs.items():
            key = (ts.metric_name, "scope", attr_name, str(attr_value))
            if key not in metadata_map:
                metadata_map[key] = MetricsMetadata(
                    metric_name=ts.metric_name,
                    attr_name=attr_name,
                    attr_type="scope",
                    attr_datatype="String",
                    attr_string_value=str(attr_value),
                    timestamp=metric.timestamp,
                    temporality=ts.temporality,
                    description=ts.description,
                    unit=ts.unit,
                    type_=ts.type,
                    is_monotonic=ts.is_monotonic,
                )

    if len(metadata_map) > 0:
        conn.insert(
            database="signoz_metrics",
            table="distributed_metadata",
            column_names=[
                "temporality",
                "metric_name",
                "description",
                "unit",
                "type",
                "is_monotonic",
                "attr_name",
                "attr_type",
                "attr_datatype",
                "attr_string_value",
                "first_reported_unix_milli",
                "last_reported_unix_milli",
            ],
            data=[m.to_row() for m in metadata_map.values()],
        )


_METRICS_TABLES_TO_TRUNCATE = [
    "time_series_v4",
    "samples_v4",
    "exp_hist",
    "metadata",
]


def truncate_metrics_tables(conn, cluster: str) -> None:
    """Truncate all metrics tables. Used by the pytest fixture teardown and by
    the seeder's DELETE /telemetry/metrics endpoint."""
    for table in _METRICS_TABLES_TO_TRUNCATE:
        conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")


@pytest.fixture(name="insert_metrics", scope="function")
def insert_metrics(
    clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[[list[Metrics]], None], Any]:
    def _insert_metrics(metrics: list[Metrics]) -> None:
        insert_metrics_to_clickhouse(clickhouse.conn, metrics)

    yield _insert_metrics

    truncate_metrics_tables(
        clickhouse.conn,
        clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"],
    )


def insert_exp_histogram_metrics_to_clickhouse(conn, metrics: list[ExpHistogramMetrics]) -> None:
    """
    Insert exponential histograms into ClickHouse tables.
    Handles insertion into:
    - distributed_time_series_v4 (time series metadata)
    - distributed_exp_hist (per-point sketches)
    - distributed_metadata (metric attribute metadata)
    """
    insert_time_series_to_clickhouse(conn, [metric.time_series for metric in metrics])

    # `sketch` is AggregateFunction(quantilesDD(...), UInt64) — the state has to be
    # folded server-side, it cannot be sent as a literal. The quantilesDDState
    # parameters must match the column's exactly or the INSERT is rejected.
    for metric in metrics:
        hist = metric.exp_hist
        conn.command(
            "INSERT INTO signoz_metrics.distributed_exp_hist "
            "(env, temporality, metric_name, fingerprint, unix_milli, count, sum, min, max, sketch, flags) "
            "SELECT %(env)s, %(temporality)s, %(metric_name)s, %(fingerprint)s, %(unix_milli)s, "
            "%(count)s, %(sum)s, %(min)s, %(max)s, "
            "quantilesDDState(0.01, 0.5, 0.75, 0.9, 0.95, 0.99)(toUInt64(observation)), %(flags)s "
            "FROM (SELECT arrayJoin(%(observations)s) AS observation)",
            parameters={
                "env": hist.env,
                "temporality": hist.temporality,
                "metric_name": hist.metric_name,
                "fingerprint": int(hist.fingerprint),
                "unix_milli": int(hist.unix_milli),
                "count": int(hist.count),
                "sum": float(hist.sum),
                "min": float(hist.min),
                "max": float(hist.max),
                "observations": hist.observations,
                "flags": int(hist.flags),
            },
        )

    insert_metrics_metadata_to_clickhouse(conn, metrics)


@pytest.fixture(name="insert_exp_histogram_metrics", scope="function")
def insert_exp_histogram_metrics(
    clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[[list[ExpHistogramMetrics]], None], Any]:
    def _insert_exp_histogram_metrics(metrics: list[ExpHistogramMetrics]) -> None:
        insert_exp_histogram_metrics_to_clickhouse(clickhouse.conn, metrics)

    yield _insert_exp_histogram_metrics

    truncate_metrics_tables(
        clickhouse.conn,
        clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"],
    )


def insert_reduced_metrics_to_clickhouse(
    conn,
    time_series: list[MetricsReducedTimeSeries],
    last_samples: list[MetricsReducedSampleLast60s] | None = None,
    sum_samples: list[MetricsReducedSampleSum60s] | None = None,
) -> None:
    """Insert reduced series into distributed_time_series_v4_reduced and 60s
    buckets into the reduced samples tables. These tables exist only when
    the schema migrator version includes the metrics cardinality-control
    migration."""
    if time_series:
        conn.insert(
            database="signoz_metrics",
            table="distributed_time_series_v4_reduced",
            column_names=[
                "env",
                "temporality",
                "metric_name",
                "description",
                "unit",
                "type",
                "is_monotonic",
                "fingerprint",
                "unix_milli",
                "labels",
                "attrs",
                "scope_attrs",
                "resource_attrs",
            ],
            data=[ts.to_row() for ts in time_series],
        )

    if last_samples:
        conn.insert(
            database="signoz_metrics",
            table="distributed_samples_v4_reduced_last_60s",
            column_names=[
                "env",
                "temporality",
                "metric_name",
                "reduced_fingerprint",
                "unix_milli",
                "sum_last",
                "min",
                "max",
                "sum_values",
                "count_series",
                "count_samples",
                "computed_at",
            ],
            data=[sample.to_row() for sample in last_samples],
        )

    if sum_samples:
        conn.insert(
            database="signoz_metrics",
            table="distributed_samples_v4_reduced_sum_60s",
            column_names=[
                "env",
                "temporality",
                "metric_name",
                "reduced_fingerprint",
                "unix_milli",
                "sum",
                "count_series",
                "count_samples",
                "computed_at",
            ],
            data=[sample.to_row() for sample in sum_samples],
        )


def insert_buffer_metrics_to_clickhouse(
    conn,
    time_series: list[MetricsBufferTimeSeries],
    samples: list[MetricsBufferSample],
) -> None:
    if time_series:
        conn.insert(
            database="signoz_metrics",
            table="distributed_time_series_v4_buffer",
            column_names=[
                "env",
                "temporality",
                "metric_name",
                "description",
                "unit",
                "type",
                "is_monotonic",
                "fingerprint",
                "reduced_fingerprint",
                "is_reduced",
                "unix_milli",
                "labels",
                "attrs",
                "scope_attrs",
                "resource_attrs",
            ],
            data=[ts.to_row() for ts in time_series],
        )

    if samples:
        conn.insert(
            database="signoz_metrics",
            table="distributed_samples_v4_buffer",
            column_names=[
                "env",
                "temporality",
                "metric_name",
                "fingerprint",
                "reduced_fingerprint",
                "is_monotonic",
                "unix_milli",
                "value",
                "flags",
            ],
            data=[sample.to_row() for sample in samples],
        )


@pytest.fixture(name="insert_reduced_metrics", scope="function")
def insert_reduced_metrics(
    clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[..., None], Any]:
    def _insert_reduced_metrics(
        time_series: list[MetricsReducedTimeSeries],
        last_samples: list[MetricsReducedSampleLast60s] | None = None,
        sum_samples: list[MetricsReducedSampleSum60s] | None = None,
    ) -> None:
        insert_reduced_metrics_to_clickhouse(clickhouse.conn, time_series, last_samples, sum_samples)

    yield _insert_reduced_metrics

    cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
    for table in _REDUCED_METRICS_TABLES_TO_TRUNCATE:
        clickhouse.conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")


@pytest.fixture(name="insert_buffer_metrics", scope="function")
def insert_buffer_metrics(
    clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[..., None], Any]:
    def _insert_buffer_metrics(
        time_series: list[MetricsBufferTimeSeries],
        samples: list[MetricsBufferSample],
    ) -> None:
        insert_buffer_metrics_to_clickhouse(clickhouse.conn, time_series, samples)

    yield _insert_buffer_metrics

    cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
    for table in _REDUCED_METRICS_TABLES_TO_TRUNCATE:
        clickhouse.conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")


@pytest.fixture(name="remove_metrics_ttl_and_storage_settings", scope="function")
def remove_metrics_ttl_and_storage_settings(signoz: types.SigNoz):
    """
    Remove any custom TTL settings on metrics tables to revert to default retention.
    Also resets storage policy to default by recreating tables if needed.
    """
    tables = [
        "samples_v4",
        "samples_v4_agg_5m",
        "samples_v4_agg_30m",
        "time_series_v4",
        "time_series_v4_6hrs",
        "time_series_v4_1day",
        "time_series_v4_1week",
        "exp_hist",
        "metadata",
    ]

    cluster = signoz.telemetrystore.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
    for table in tables:
        try:
            signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' REMOVE TTL")
            signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' RESET SETTING storage_policy;")
        except Exception as e:  # pylint: disable=broad-exception-caught
            print(f"ttl and storage policy reset failed for {table}: {e}")