SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
import datetime
import hashlib
import json
from abc import ABC
from collections.abc import Callable, Generator
from typing import Any
import numpy as np
import pytest
from fixtures import types
from fixtures.time import parse_timestamp
_REDUCED_METRICS_TABLES_TO_TRUNCATE = [
"time_series_v4_reduced",
"samples_v4_reduced_last_60s",
"samples_v4_reduced_sum_60s",
"time_series_v4_buffer",
"samples_v4_buffer",
]
class MetricsTimeSeries(ABC):
"""Represents a row in the time_series_v4 table."""
env: str
temporality: str
metric_name: str
description: str
unit: str
type: str
is_monotonic: bool
fingerprint: np.uint64
unix_milli: np.int64
labels: str
attrs: dict[str, str]
scope_attrs: dict[str, str]
resource_attrs: dict[str, str]
def __init__(
self,
metric_name: str,
labels: dict[str, str],
timestamp: datetime.datetime,
temporality: str = "Unspecified",
description: str = "",
unit: str = "",
type_: str = "Sum",
is_monotonic: bool = True,
env: str = "default",
resource_attrs: dict[str, str] = {},
scope_attrs: dict[str, str] = {},
) -> None:
# Create a copy of labels to avoid mutating the caller's dictionary
labels = dict(labels)
# Add metric_name to the labels to support promql queries
labels["__name__"] = metric_name
self.env = env
self.metric_name = metric_name
self.temporality = temporality
self.description = description
self.unit = unit
self.type = type_
self.is_monotonic = is_monotonic
self.labels = json.dumps(labels, separators=(",", ":"))
self.attrs = labels
self.scope_attrs = scope_attrs
self.resource_attrs = resource_attrs
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
# Calculate fingerprint from metric_name + labels
fingerprint_str = metric_name + self.labels
self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.description,
self.unit,
self.type,
self.is_monotonic,
self.fingerprint,
self.unix_milli,
self.labels,
self.attrs,
self.scope_attrs,
self.resource_attrs,
]
class MetricsSample(ABC):
"""Represents a row in the samples_v4 table."""
env: str
temporality: str
metric_name: str
fingerprint: np.uint64
unix_milli: np.int64
value: np.float64
flags: np.uint32
def __init__(
self,
metric_name: str,
fingerprint: np.uint64,
timestamp: datetime.datetime,
value: float,
temporality: str = "Unspecified",
env: str = "default",
flags: int = 0,
) -> None:
self.env = env
self.temporality = temporality
self.metric_name = metric_name
self.fingerprint = fingerprint
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
self.value = np.float64(value)
self.flags = np.uint32(flags)
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.fingerprint,
self.unix_milli,
self.value,
self.flags,
]
class MetricsExpHist(ABC):
"""Represents a row in the exp_hist table for exponential histograms.
`observations` must be non-empty; ClickHouse folds it into the `sketch`
AggregateFunction state on insert.
TODO: take an ExponentialHistogramDataPoint and compute the sketch bytes the
way the collector does, so the fixture exercises the real write path instead
of having ClickHouse build the state.
"""
env: str
temporality: str
metric_name: str
fingerprint: np.uint64
unix_milli: np.int64
count: np.uint64
sum: np.float64
min: np.float64
max: np.float64
observations: list[int]
flags: np.uint32
def __init__(
self,
metric_name: str,
fingerprint: np.uint64,
timestamp: datetime.datetime,
observations: list[int],
temporality: str = "Unspecified",
env: str = "default",
flags: int = 0,
) -> None:
self.env = env
self.temporality = temporality
self.metric_name = metric_name
self.fingerprint = fingerprint
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
self.observations = observations
self.count = np.uint64(len(observations))
self.sum = np.float64(sum(observations))
self.min = np.float64(min(observations))
self.max = np.float64(max(observations))
self.flags = np.uint32(flags)
class MetricsMetadata(ABC):
"""Represents a row in the metadata table for metric metadata."""
temporality: str
metric_name: str
description: str
unit: str
type: str
is_monotonic: bool
attr_name: str
attr_type: str
attr_datatype: str
attr_string_value: str
first_reported_unix_milli: np.int64
last_reported_unix_milli: np.int64
def __init__(
self,
metric_name: str,
attr_name: str,
attr_type: str,
attr_datatype: str,
attr_string_value: str,
timestamp: datetime.datetime,
temporality: str = "Unspecified",
description: str = "",
unit: str = "",
type_: str = "Sum",
is_monotonic: bool = True,
) -> None:
self.temporality = temporality
self.metric_name = metric_name
self.description = description
self.unit = unit
self.type = type_
self.is_monotonic = is_monotonic
self.attr_name = attr_name
self.attr_type = attr_type
self.attr_datatype = attr_datatype
self.attr_string_value = attr_string_value
unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
self.first_reported_unix_milli = unix_milli
self.last_reported_unix_milli = unix_milli
def to_row(self) -> list:
return [
self.temporality,
self.metric_name,
self.description,
self.unit,
self.type,
self.is_monotonic,
self.attr_name,
self.attr_type,
self.attr_datatype,
self.attr_string_value,
self.first_reported_unix_milli,
self.last_reported_unix_milli,
]
class Metrics(ABC):
"""High-level metric representation. Produces both time series and sample entries."""
metric_name: str
labels: dict[str, str]
temporality: str
timestamp: datetime.datetime
value: float
flags: int
@property
def time_series(self) -> MetricsTimeSeries:
return self._time_series
@property
def sample(self) -> MetricsSample:
return self._sample
def __init__(
self,
metric_name: str,
labels: dict[str, str] = {},
timestamp: datetime.datetime | None = None,
value: float = 0.0,
temporality: str = "Unspecified",
flags: int = 0,
description: str = "",
unit: str = "",
type_: str = "Sum",
is_monotonic: bool = True,
env: str = "default",
resource_attributes: dict[str, str] = {},
scope_attributes: dict[str, str] = {},
) -> None:
if timestamp is None:
timestamp = datetime.datetime.now()
self.metric_name = metric_name
self.labels = labels
self.temporality = temporality
self.timestamp = timestamp
self.value = value
self.flags = flags
self._time_series = MetricsTimeSeries(
metric_name=metric_name,
labels=labels,
timestamp=timestamp,
temporality=temporality,
description=description,
unit=unit,
type_=type_,
is_monotonic=is_monotonic,
env=env,
resource_attrs=resource_attributes,
scope_attrs=scope_attributes,
)
self._sample = MetricsSample(
metric_name=metric_name,
fingerprint=self._time_series.fingerprint,
timestamp=timestamp,
value=value,
temporality=temporality,
env=env,
flags=flags,
)
def to_dict(self) -> dict:
return {
"metric_name": self.metric_name,
"labels": self.labels,
"timestamp": self.timestamp.isoformat(),
"value": self.value,
"temporality": self.temporality,
"type_": self._time_series.type,
"is_monotonic": self._time_series.is_monotonic,
"flags": self.flags,
"description": self._time_series.description,
"unit": self._time_series.unit,
"env": self._time_series.env,
"resource_attrs": self._time_series.resource_attrs,
"scope_attrs": self._time_series.scope_attrs,
}
@classmethod
def from_dict(
cls,
data: dict,
# base_time: Optional[datetime.datetime] = None,
metric_name_override: str | None = None,
) -> "Metrics":
"""
Create a Metrics instance from a dict.
Args:
data: The dict containing metric data
base_time: If provided, timestamps are shifted relative to this time.
The earliest timestamp in the data becomes base_time.
metric_name_override: If provided, overrides the metric_name from data
"""
# parse timestamp from iso format
timestamp = parse_timestamp(data["timestamp"])
return cls(
metric_name=metric_name_override or data["metric_name"],
labels=data.get("labels", {}),
timestamp=timestamp,
value=data["value"],
temporality=data.get("temporality", "Unspecified"),
flags=data.get("flags", 0),
description=data.get("description", ""),
unit=data.get("unit", ""),
type_=data.get("type_", "Sum"),
is_monotonic=data.get("is_monotonic", True),
env=data.get("env", "default"),
resource_attributes=data.get("resource_attrs", {}),
scope_attributes=data.get("scope_attrs", {}),
)
@classmethod
def load_from_file(
cls,
file_path: str,
base_time: datetime.datetime | None = None,
metric_name_override: str | None = None,
label_substitutions: dict[str, str] | None = None,
) -> list["Metrics"]:
"""
Load metrics from a JSONL file.
Each line should be a JSON object representing a metric.
Args:
file_path: Path to the JSONL file
base_time: If provided, all timestamps are shifted so the earliest
timestamp in the file maps to base_time
metric_name_override: If provided, overrides metric_name for all metrics
label_substitutions: If provided, any label whose value equals a key is
rewritten to that key's value (placeholder substitution,
e.g. {"__START_TIME__": start_time.isoformat()})
"""
data_list = []
with open(file_path, encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
data = json.loads(line)
if label_substitutions:
labels = data.get("labels", {})
for key, value in labels.items():
if value in label_substitutions:
labels[key] = label_substitutions[value]
data_list.append(data)
if not data_list:
return []
# If base_time provided, calculate time offset
time_offset = datetime.timedelta(0)
if base_time is not None:
# Find earliest timestamp
earliest = None
for data in data_list:
ts = parse_timestamp(data["timestamp"])
if earliest is None or ts < earliest:
earliest = ts
if earliest is not None:
time_offset = base_time - earliest
metrics = []
for data in data_list:
original_ts = parse_timestamp(data["timestamp"])
adjusted_ts = original_ts + time_offset
data["timestamp"] = adjusted_ts.isoformat()
metrics.append(cls.from_dict(data, metric_name_override=metric_name_override))
return metrics
class ExpHistogramMetrics(ABC):
"""High-level exponential histogram representation. Produces both time series
and exp_hist entries."""
metric_name: str
labels: dict[str, str]
temporality: str
timestamp: datetime.datetime
observations: list[int]
@property
def time_series(self) -> MetricsTimeSeries:
return self._time_series
@property
def exp_hist(self) -> MetricsExpHist:
return self._exp_hist
def __init__(
self,
metric_name: str,
observations: list[int],
labels: dict[str, str] = {},
timestamp: datetime.datetime | None = None,
temporality: str = "Delta",
flags: int = 0,
description: str = "",
unit: str = "",
env: str = "default",
resource_attributes: dict[str, str] = {},
scope_attributes: dict[str, str] = {},
) -> None:
if timestamp is None:
timestamp = datetime.datetime.now()
self.metric_name = metric_name
self.labels = labels
self.temporality = temporality
self.timestamp = timestamp
self.observations = observations
self._time_series = MetricsTimeSeries(
metric_name=metric_name,
labels=labels,
timestamp=timestamp,
temporality=temporality,
description=description,
unit=unit,
# the querier resolves the metric type from this column, and only an
# ExponentialHistogram here routes the query to the sketch read
type_="ExponentialHistogram",
is_monotonic=False,
env=env,
resource_attrs=resource_attributes,
scope_attrs=scope_attributes,
)
self._exp_hist = MetricsExpHist(
metric_name=metric_name,
fingerprint=self._time_series.fingerprint,
timestamp=timestamp,
observations=observations,
temporality=temporality,
env=env,
flags=flags,
)
class MetricsReducedTimeSeries(ABC):
"""Represents a row in the time_series_v4_reduced table i.e what
the time_series_v4_reduced_mv materializes for a metric under a
reduction rule. One row per kept-label group. `fingerprint` holds the
reduced fingerprint and `labels` contains only the kept labels.
The fingerprint recipe (md5, like MetricsTimeSeries) does not match the
collector's real hash; it only needs to be consistent with the
reduced_fingerprint used in the reduced samples rows.
"""
def __init__( # pylint: disable=too-many-arguments
self,
metric_name: str,
kept_labels: dict[str, str],
timestamp: datetime.datetime,
temporality: str = "Unspecified",
description: str = "",
unit: str = "",
type_: str = "Gauge",
is_monotonic: bool = False,
env: str = "default",
) -> None:
kept_labels = dict(kept_labels)
kept_labels["__name__"] = metric_name
self.env = env
# mirror time_series_v4_reduced_mv: monotonic cumulative counters are
# reduced as deltas
if temporality == "Cumulative" and is_monotonic:
temporality = "Delta"
self.temporality = temporality
self.metric_name = metric_name
self.description = description
self.unit = unit
self.type = type_
self.is_monotonic = is_monotonic
self.labels = json.dumps(kept_labels, separators=(",", ":"))
self.attrs = kept_labels
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3) // 3600000 * 3600000)
fingerprint_str = metric_name + self.labels
self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.description,
self.unit,
self.type,
self.is_monotonic,
self.fingerprint,
self.unix_milli,
self.labels,
self.attrs,
{},
{},
]
class MetricsReducedSampleLast60s(ABC):
"""Represents a row in the samples_v4_reduced_last_60s table. One 60s
bucket per reduced group, as the samples_v4_reduced_last_60s_mv refresh
would emit it (gauges and non-monotonic cumulative sums)."""
def __init__( # pylint: disable=too-many-arguments
self,
metric_name: str,
reduced_fingerprint: np.uint64,
timestamp: datetime.datetime,
sum_last: float,
min_value: float,
max_value: float,
sum_values: float,
count_series: int,
count_samples: int,
temporality: str = "Unspecified",
env: str = "default",
computed_at: datetime.datetime | None = None,
) -> None:
self.env = env
self.temporality = temporality
self.metric_name = metric_name
self.reduced_fingerprint = reduced_fingerprint
# buckets are 60s-aligned: intDiv(unix_milli, 60000) * 60000
self.unix_milli = np.int64((int(timestamp.timestamp() * 1e3) // 60000) * 60000)
self.sum_last = np.float64(sum_last)
self.min = np.float64(min_value)
self.max = np.float64(max_value)
self.sum_values = np.float64(sum_values)
self.count_series = np.uint64(count_series)
self.count_samples = np.uint64(count_samples)
# the refresh stamps now(); default to shortly after the bucket closes
if computed_at is None:
computed_at = datetime.datetime.fromtimestamp(int(self.unix_milli) / 1e3, tz=datetime.UTC) + datetime.timedelta(seconds=180)
self.computed_at = computed_at
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.reduced_fingerprint,
self.unix_milli,
self.sum_last,
self.min,
self.max,
self.sum_values,
self.count_series,
self.count_samples,
self.computed_at,
]
class MetricsReducedSampleSum60s(ABC):
"""Represents a row in the samples_v4_reduced_sum_60s table. One 60s
bucket per reduced group for delta counters and histograms."""
def __init__( # pylint: disable=too-many-arguments
self,
metric_name: str,
reduced_fingerprint: np.uint64,
timestamp: datetime.datetime,
sum_value: float,
count_series: int,
count_samples: int,
temporality: str = "Delta",
env: str = "default",
computed_at: datetime.datetime | None = None,
) -> None:
self.env = env
self.temporality = temporality
self.metric_name = metric_name
self.reduced_fingerprint = reduced_fingerprint
self.unix_milli = np.int64((int(timestamp.timestamp() * 1e3) // 60000) * 60000)
self.sum = np.float64(sum_value)
self.count_series = np.uint64(count_series)
self.count_samples = np.uint64(count_samples)
if computed_at is None:
computed_at = datetime.datetime.fromtimestamp(int(self.unix_milli) / 1e3, tz=datetime.UTC) + datetime.timedelta(seconds=180)
self.computed_at = computed_at
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.reduced_fingerprint,
self.unix_milli,
self.sum,
self.count_series,
self.count_samples,
self.computed_at,
]
class MetricsBufferTimeSeries(ABC):
"""Represents a row in the time_series_v4_buffer table. This is the collector's
universal landing target under cardinality control. For a ruled metric the
collector writes two rows per series: the raw one (is_reduced=false, full
labels, reduced_fingerprint pointing at its group) and the group's reduced
one (is_reduced=true, kept labels, fingerprint = reduced fingerprint)."""
def __init__( # pylint: disable=too-many-arguments
self,
metric_name: str,
labels: dict[str, str],
timestamp: datetime.datetime,
reduced_fingerprint: np.uint64 | int = 0,
is_reduced: bool = False,
temporality: str = "Unspecified",
description: str = "",
unit: str = "",
type_: str = "Gauge",
is_monotonic: bool = False,
env: str = "default",
) -> None:
labels = dict(labels)
labels["__name__"] = metric_name
self.env = env
self.temporality = temporality
self.metric_name = metric_name
self.description = description
self.unit = unit
self.type = type_
self.is_monotonic = is_monotonic
self.reduced_fingerprint = np.uint64(reduced_fingerprint)
self.is_reduced = is_reduced
self.labels = json.dumps(labels, separators=(",", ":"))
self.attrs = labels
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3) // 3600000 * 3600000)
fingerprint_str = metric_name + self.labels
self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.description,
self.unit,
self.type,
self.is_monotonic,
self.fingerprint,
self.reduced_fingerprint,
self.is_reduced,
self.unix_milli,
self.labels,
self.attrs,
{},
{},
]
class MetricsBufferSample(ABC):
"""Represents a row in the samples_v4_buffer table. Ruled samples carry
the raw fingerprint plus the group's reduced_fingerprint; unruled samples
have reduced_fingerprint = 0."""
def __init__( # pylint: disable=too-many-arguments
self,
metric_name: str,
fingerprint: np.uint64,
timestamp: datetime.datetime,
value: float,
reduced_fingerprint: np.uint64 | int = 0,
is_monotonic: bool = False,
temporality: str = "Unspecified",
env: str = "default",
flags: int = 0,
) -> None:
self.env = env
self.temporality = temporality
self.metric_name = metric_name
self.fingerprint = fingerprint
self.reduced_fingerprint = np.uint64(reduced_fingerprint)
self.is_monotonic = is_monotonic
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
self.value = np.float64(value)
self.flags = np.uint32(flags)
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.fingerprint,
self.reduced_fingerprint,
self.is_monotonic,
self.unix_milli,
self.value,
self.flags,
]
def insert_time_series_to_clickhouse(conn, time_series: list[MetricsTimeSeries]) -> None:
"""
Insert one distributed_time_series_v4 registration row per (series, hour
bucket), unix_milli floored to the hour — the exporter's exact shape.
Readers floor lookup windows to these buckets: skipping per-bucket
re-registration or keeping raw mid-hour timestamps hides series in ways
production never sees.
"""
time_series_map: dict[tuple[int, int], MetricsTimeSeries] = {}
for ts in time_series:
fp = int(ts.fingerprint)
hour_bucket = int(ts.unix_milli) // 3_600_000
if (fp, hour_bucket) not in time_series_map:
ts.unix_milli = np.int64(hour_bucket * 3_600_000)
time_series_map[(fp, hour_bucket)] = ts
if len(time_series_map) == 0:
return
conn.insert(
database="signoz_metrics",
table="distributed_time_series_v4",
column_names=[
"env",
"temporality",
"metric_name",
"description",
"unit",
"type",
"is_monotonic",
"fingerprint",
"unix_milli",
"labels",
"attrs",
"scope_attrs",
"resource_attrs",
],
data=[ts.to_row() for ts in time_series_map.values()],
)
def insert_metrics_to_clickhouse(conn, metrics: list[Metrics]) -> None:
"""
Insert metrics into ClickHouse tables.
Handles insertion into:
- distributed_time_series_v4 (time series metadata)
- distributed_samples_v4 (actual sample values)
- distributed_metadata (metric attribute metadata)
Pure function so the seeder container can reuse the exact insert path
used by the pytest fixture. `conn` is a clickhouse-connect Client.
"""
insert_time_series_to_clickhouse(conn, [metric.time_series for metric in metrics])
samples = [metric.sample for metric in metrics]
if len(samples) > 0:
conn.insert(
database="signoz_metrics",
table="distributed_samples_v4",
column_names=[
"env",
"temporality",
"metric_name",
"fingerprint",
"unix_milli",
"value",
"flags",
],
data=[sample.to_row() for sample in samples],
)
insert_metrics_metadata_to_clickhouse(conn, metrics)
def insert_metrics_metadata_to_clickhouse(conn, metrics: list) -> None:
"""
Insert the distributed_metadata rows describing each metric's point, resource
and scope attributes. Accepts anything exposing `time_series`, `labels` and
`timestamp`.
"""
# (metric_name, attr_type, attr_name, attr_value) -> MetricsMetadata
metadata_map: dict[tuple, MetricsMetadata] = {}
for metric in metrics:
ts = metric.time_series
for attr_name, attr_value in metric.labels.items():
key = (ts.metric_name, "point", attr_name, str(attr_value))
if key not in metadata_map:
metadata_map[key] = MetricsMetadata(
metric_name=ts.metric_name,
attr_name=attr_name,
attr_type="point",
attr_datatype="String",
attr_string_value=str(attr_value),
timestamp=metric.timestamp,
temporality=ts.temporality,
description=ts.description,
unit=ts.unit,
type_=ts.type,
is_monotonic=ts.is_monotonic,
)
for attr_name, attr_value in ts.resource_attrs.items():
key = (ts.metric_name, "resource", attr_name, str(attr_value))
if key not in metadata_map:
metadata_map[key] = MetricsMetadata(
metric_name=ts.metric_name,
attr_name=attr_name,
attr_type="resource",
attr_datatype="String",
attr_string_value=str(attr_value),
timestamp=metric.timestamp,
temporality=ts.temporality,
description=ts.description,
unit=ts.unit,
type_=ts.type,
is_monotonic=ts.is_monotonic,
)
for attr_name, attr_value in ts.scope_attrs.items():
key = (ts.metric_name, "scope", attr_name, str(attr_value))
if key not in metadata_map:
metadata_map[key] = MetricsMetadata(
metric_name=ts.metric_name,
attr_name=attr_name,
attr_type="scope",
attr_datatype="String",
attr_string_value=str(attr_value),
timestamp=metric.timestamp,
temporality=ts.temporality,
description=ts.description,
unit=ts.unit,
type_=ts.type,
is_monotonic=ts.is_monotonic,
)
if len(metadata_map) > 0:
conn.insert(
database="signoz_metrics",
table="distributed_metadata",
column_names=[
"temporality",
"metric_name",
"description",
"unit",
"type",
"is_monotonic",
"attr_name",
"attr_type",
"attr_datatype",
"attr_string_value",
"first_reported_unix_milli",
"last_reported_unix_milli",
],
data=[m.to_row() for m in metadata_map.values()],
)
_METRICS_TABLES_TO_TRUNCATE = [
"time_series_v4",
"samples_v4",
"exp_hist",
"metadata",
]
def truncate_metrics_tables(conn, cluster: str) -> None:
"""Truncate all metrics tables. Used by the pytest fixture teardown and by
the seeder's DELETE /telemetry/metrics endpoint."""
for table in _METRICS_TABLES_TO_TRUNCATE:
conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")
@pytest.fixture(name="insert_metrics", scope="function")
def insert_metrics(
clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[[list[Metrics]], None], Any]:
def _insert_metrics(metrics: list[Metrics]) -> None:
insert_metrics_to_clickhouse(clickhouse.conn, metrics)
yield _insert_metrics
truncate_metrics_tables(
clickhouse.conn,
clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"],
)
def insert_exp_histogram_metrics_to_clickhouse(conn, metrics: list[ExpHistogramMetrics]) -> None:
"""
Insert exponential histograms into ClickHouse tables.
Handles insertion into:
- distributed_time_series_v4 (time series metadata)
- distributed_exp_hist (per-point sketches)
- distributed_metadata (metric attribute metadata)
"""
insert_time_series_to_clickhouse(conn, [metric.time_series for metric in metrics])
# `sketch` is AggregateFunction(quantilesDD(...), UInt64) — the state has to be
# folded server-side, it cannot be sent as a literal. The quantilesDDState
# parameters must match the column's exactly or the INSERT is rejected.
for metric in metrics:
hist = metric.exp_hist
conn.command(
"INSERT INTO signoz_metrics.distributed_exp_hist "
"(env, temporality, metric_name, fingerprint, unix_milli, count, sum, min, max, sketch, flags) "
"SELECT %(env)s, %(temporality)s, %(metric_name)s, %(fingerprint)s, %(unix_milli)s, "
"%(count)s, %(sum)s, %(min)s, %(max)s, "
"quantilesDDState(0.01, 0.5, 0.75, 0.9, 0.95, 0.99)(toUInt64(observation)), %(flags)s "
"FROM (SELECT arrayJoin(%(observations)s) AS observation)",
parameters={
"env": hist.env,
"temporality": hist.temporality,
"metric_name": hist.metric_name,
"fingerprint": int(hist.fingerprint),
"unix_milli": int(hist.unix_milli),
"count": int(hist.count),
"sum": float(hist.sum),
"min": float(hist.min),
"max": float(hist.max),
"observations": hist.observations,
"flags": int(hist.flags),
},
)
insert_metrics_metadata_to_clickhouse(conn, metrics)
@pytest.fixture(name="insert_exp_histogram_metrics", scope="function")
def insert_exp_histogram_metrics(
clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[[list[ExpHistogramMetrics]], None], Any]:
def _insert_exp_histogram_metrics(metrics: list[ExpHistogramMetrics]) -> None:
insert_exp_histogram_metrics_to_clickhouse(clickhouse.conn, metrics)
yield _insert_exp_histogram_metrics
truncate_metrics_tables(
clickhouse.conn,
clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"],
)
def insert_reduced_metrics_to_clickhouse(
conn,
time_series: list[MetricsReducedTimeSeries],
last_samples: list[MetricsReducedSampleLast60s] | None = None,
sum_samples: list[MetricsReducedSampleSum60s] | None = None,
) -> None:
"""Insert reduced series into distributed_time_series_v4_reduced and 60s
buckets into the reduced samples tables. These tables exist only when
the schema migrator version includes the metrics cardinality-control
migration."""
if time_series:
conn.insert(
database="signoz_metrics",
table="distributed_time_series_v4_reduced",
column_names=[
"env",
"temporality",
"metric_name",
"description",
"unit",
"type",
"is_monotonic",
"fingerprint",
"unix_milli",
"labels",
"attrs",
"scope_attrs",
"resource_attrs",
],
data=[ts.to_row() for ts in time_series],
)
if last_samples:
conn.insert(
database="signoz_metrics",
table="distributed_samples_v4_reduced_last_60s",
column_names=[
"env",
"temporality",
"metric_name",
"reduced_fingerprint",
"unix_milli",
"sum_last",
"min",
"max",
"sum_values",
"count_series",
"count_samples",
"computed_at",
],
data=[sample.to_row() for sample in last_samples],
)
if sum_samples:
conn.insert(
database="signoz_metrics",
table="distributed_samples_v4_reduced_sum_60s",
column_names=[
"env",
"temporality",
"metric_name",
"reduced_fingerprint",
"unix_milli",
"sum",
"count_series",
"count_samples",
"computed_at",
],
data=[sample.to_row() for sample in sum_samples],
)
def insert_buffer_metrics_to_clickhouse(
conn,
time_series: list[MetricsBufferTimeSeries],
samples: list[MetricsBufferSample],
) -> None:
if time_series:
conn.insert(
database="signoz_metrics",
table="distributed_time_series_v4_buffer",
column_names=[
"env",
"temporality",
"metric_name",
"description",
"unit",
"type",
"is_monotonic",
"fingerprint",
"reduced_fingerprint",
"is_reduced",
"unix_milli",
"labels",
"attrs",
"scope_attrs",
"resource_attrs",
],
data=[ts.to_row() for ts in time_series],
)
if samples:
conn.insert(
database="signoz_metrics",
table="distributed_samples_v4_buffer",
column_names=[
"env",
"temporality",
"metric_name",
"fingerprint",
"reduced_fingerprint",
"is_monotonic",
"unix_milli",
"value",
"flags",
],
data=[sample.to_row() for sample in samples],
)
@pytest.fixture(name="insert_reduced_metrics", scope="function")
def insert_reduced_metrics(
clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[..., None], Any]:
def _insert_reduced_metrics(
time_series: list[MetricsReducedTimeSeries],
last_samples: list[MetricsReducedSampleLast60s] | None = None,
sum_samples: list[MetricsReducedSampleSum60s] | None = None,
) -> None:
insert_reduced_metrics_to_clickhouse(clickhouse.conn, time_series, last_samples, sum_samples)
yield _insert_reduced_metrics
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
for table in _REDUCED_METRICS_TABLES_TO_TRUNCATE:
clickhouse.conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")
@pytest.fixture(name="insert_buffer_metrics", scope="function")
def insert_buffer_metrics(
clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[..., None], Any]:
def _insert_buffer_metrics(
time_series: list[MetricsBufferTimeSeries],
samples: list[MetricsBufferSample],
) -> None:
insert_buffer_metrics_to_clickhouse(clickhouse.conn, time_series, samples)
yield _insert_buffer_metrics
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
for table in _REDUCED_METRICS_TABLES_TO_TRUNCATE:
clickhouse.conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")
@pytest.fixture(name="remove_metrics_ttl_and_storage_settings", scope="function")
def remove_metrics_ttl_and_storage_settings(signoz: types.SigNoz):
"""
Remove any custom TTL settings on metrics tables to revert to default retention.
Also resets storage policy to default by recreating tables if needed.
"""
tables = [
"samples_v4",
"samples_v4_agg_5m",
"samples_v4_agg_30m",
"time_series_v4",
"time_series_v4_6hrs",
"time_series_v4_1day",
"time_series_v4_1week",
"exp_hist",
"metadata",
]
cluster = signoz.telemetrystore.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
for table in tables:
try:
signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' REMOVE TTL")
signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' RESET SETTING storage_policy;")
except Exception as e: # pylint: disable=broad-exception-caught
print(f"ttl and storage policy reset failed for {table}: {e}")