SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
import datetime
import hashlib
import json
import secrets
import uuid
from abc import ABC
from collections.abc import Callable, Generator
from enum import Enum
from typing import Any, Literal
from urllib.parse import urlparse
import numpy as np
import pytest
from fixtures import types
from fixtures.fingerprint import LogsOrTracesFingerprint
from fixtures.time import parse_duration, parse_timestamp
# All keys returned by the trace list endpoint when selectFields is empty:
# every intrinsic and calculated column, plus the merged `attributes` and
# `resource` maps that wrap the contextual columns in the response layer.
ALL_SELECT_FIELDS = [
# all intrinsic columns
"timestamp",
"trace_id",
"span_id",
"trace_state",
"parent_span_id",
"flags",
"name",
"kind",
"kind_string",
"duration_nano",
"status_code",
"status_message",
"status_code_string",
"events",
"links",
# all calculated columns
"response_status_code",
"external_http_url",
"http_url",
"external_http_method",
"http_method",
"http_host",
"db_name",
"db_operation",
"has_error",
"is_remote",
# all contextual columns (merged in response layer)
"attributes",
"resource",
"scope",
]
class TracesKind(Enum):
SPAN_KIND_UNSPECIFIED = 0
SPAN_KIND_INTERNAL = 1
SPAN_KIND_SERVER = 2
SPAN_KIND_CLIENT = 3
SPAN_KIND_PRODUCER = 4
SPAN_KIND_CONSUMER = 5
@classmethod
def from_value(cls, value: int) -> "TracesKind":
return cls(value)
def kind_string(self) -> str:
"""The `kind_string` column value, mirroring ptrace.SpanKind.String() — the exporter
writes `otelSpan.Kind().String()`. Features filter on these, e.g. third-party-apis
requires `kind_string = 'Client'`."""
return _KIND_STRINGS[self]
_KIND_STRINGS = {
TracesKind.SPAN_KIND_UNSPECIFIED: "Unspecified",
TracesKind.SPAN_KIND_INTERNAL: "Internal",
TracesKind.SPAN_KIND_SERVER: "Server",
TracesKind.SPAN_KIND_CLIENT: "Client",
TracesKind.SPAN_KIND_PRODUCER: "Producer",
TracesKind.SPAN_KIND_CONSUMER: "Consumer",
}
class TracesStatusCode(Enum):
STATUS_CODE_UNSET = 0
STATUS_CODE_OK = 1
STATUS_CODE_ERROR = 2
@classmethod
def from_value(cls, value: int) -> "TracesStatusCode":
return cls(value)
class TracesRefType(Enum):
REF_TYPE_CHILD_OF = "CHILD_OF"
REF_TYPE_FOLLOWS_FROM = "FOLLOWS_FROM"
class TraceIdGenerator(ABC):
@staticmethod
def trace_id() -> str:
return secrets.token_hex(16)
@staticmethod
def span_id() -> str:
return secrets.token_hex(8)
class TracesResource(ABC):
labels: str
fingerprint: str
seen_at_ts_bucket_start: np.int64
def __init__(
self,
labels: dict[str, str],
fingerprint: str,
seen_at_ts_bucket_start: np.int64,
) -> None:
self.labels = json.dumps(labels, separators=(",", ":")) # clickhouse treats {"a": "b"} differently from {"a":"b"}. In the first case it is not able to run json functions
self.fingerprint = fingerprint
self.seen_at_ts_bucket_start = seen_at_ts_bucket_start
def np_arr(self) -> np.array:
return np.array([self.labels, self.fingerprint, self.seen_at_ts_bucket_start])
class TracesResourceOrAttributeKeys(ABC):
name: str
datatype: str
tag_type: str
is_column: bool
def __init__(self, name: str, datatype: str, tag_type: str, is_column: bool = False) -> None:
self.name = name
self.datatype = datatype
self.tag_type = tag_type
self.is_column = is_column
def np_arr(self) -> np.array:
return np.array([self.name, self.tag_type, self.datatype, self.is_column], dtype=object)
class TracesTagAttributes(ABC):
unix_milli: np.int64
tag_key: str
tag_type: str
tag_data_type: str
string_value: str
number_value: np.float64
def __init__(
self,
timestamp: datetime.datetime,
tag_key: str,
tag_type: str,
tag_data_type: str,
string_value: str | None,
number_value: np.float64,
) -> None:
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
self.tag_key = tag_key
self.tag_type = tag_type
self.tag_data_type = tag_data_type
self.string_value = string_value or ""
self.number_value = number_value
def np_arr(self) -> np.array:
return np.array(
[
self.unix_milli,
self.tag_key,
self.tag_type,
self.tag_data_type,
self.string_value,
self.number_value,
]
)
class TracesEvent(ABC):
name: str
time_unix_nano: np.uint64
attribute_map: dict[str, str]
def __init__(
self,
name: str,
timestamp: datetime.datetime,
attribute_map: dict[str, str] = {},
) -> None:
self.name = name
self.time_unix_nano = np.uint64(int(timestamp.timestamp() * 1e9))
self.attribute_map = attribute_map
def np_arr(self) -> np.array:
return np.array([self.name, self.time_unix_nano, json.dumps(self.attribute_map)])
class TracesErrorEvent(ABC):
event: TracesEvent
error_id: str
error_group_id: str
trace_id: str
span_id: str
service_name: str
resources: dict[str, str]
def __init__(
self,
event: TracesEvent,
error_id: str = "",
error_group_id: str = "",
trace_id: str = "",
span_id: str = "",
service_name: str = "",
resources: dict[str, str] | None = None,
) -> None:
self.event = event
self.error_id = error_id
self.error_group_id = error_group_id
self.trace_id = trace_id
self.span_id = span_id
self.service_name = service_name
self.resources = resources or {}
def np_arr(self) -> np.array:
attrs = self.event.attribute_map
return np.array(
[
datetime.datetime.fromtimestamp(int(self.event.time_unix_nano) / 1e9, tz=datetime.UTC),
self.error_id,
self.error_group_id,
self.trace_id,
self.span_id,
self.service_name,
attrs.get("exception.type", ""),
attrs.get("exception.message", ""),
attrs.get("exception.stacktrace", ""),
str(attrs.get("exception.escaped", "")).lower() == "true",
dict(self.resources),
],
dtype=object,
)
# Column order of TracesErrorEvent.np_arr.
ERROR_INDEX_COLUMNS = [
"timestamp",
"errorID",
"groupID",
"traceID",
"spanID",
"serviceName",
"exceptionType",
"exceptionMessage",
"exceptionStacktrace",
"exceptionEscaped",
"resourceTagsMap",
]
class TracesSpanAttribute(ABC):
key: str
tag_type: str
data_type: str
string_value: str
number_value: np.float64
is_column: bool
def __init__(
self,
key: str,
tag_type: str,
data_type: str,
string_value: str = "",
number_value: np.float64 = None,
is_column: bool = False,
) -> None:
self.key = key
self.tag_type = tag_type
self.data_type = data_type
self.string_value = string_value
self.number_value = number_value
self.is_column = is_column
class TracesLink(ABC):
trace_id: str
span_id: str
ref_type: TracesRefType
def __init__(self, trace_id: str, span_id: str, ref_type: TracesRefType) -> None:
self.trace_id = trace_id
self.span_id = span_id
self.ref_type = ref_type
def __dict__(self) -> dict[str, Any]:
return {
"traceId": self.trace_id,
"spanId": self.span_id,
"refType": self.ref_type.value,
}
class Traces(ABC):
ts_bucket_start: np.uint64
resource_fingerprint: str
timestamp: np.datetime64
trace_id: str
span_id: str
trace_state: str
parent_span_id: str
flags: np.uint32
name: str
kind: np.int8
kind_string: str
duration_nano: np.uint64
status_code: np.int16
status_message: str
status_code_string: str
attribute_string: dict[str, str]
attributes_number: dict[str, np.float64]
attributes_bool: dict[str, bool]
resources_string: dict[str, str]
# Accepting parsed events and links, but will be stored as list[str], str in db
events: list[dict[str, Any]]
links: list[dict[str, Any]]
resource_json: dict[str, str]
attributes_json: dict[str, Any]
response_status_code: str
external_http_url: str
http_url: str
external_http_method: str
http_method: str
http_host: str
db_name: str
db_operation: str
has_error: bool
is_remote: str
scope_json: dict[str, Any]
resource: list[TracesResource]
tag_attributes: list[TracesTagAttributes]
resource_keys: list[TracesResourceOrAttributeKeys]
attribute_keys: list[TracesResourceOrAttributeKeys]
span_attributes: list[TracesSpanAttribute]
error_events: list[TracesErrorEvent]
def __init__(
self,
timestamp: datetime.datetime | None = None,
duration: datetime.timedelta = datetime.timedelta(seconds=1),
trace_id: str = "",
span_id: str = "",
parent_span_id: str = "",
name: str = "default span",
kind: TracesKind = TracesKind.SPAN_KIND_INTERNAL,
status_code: TracesStatusCode = TracesStatusCode.STATUS_CODE_UNSET,
status_message: str = "",
resources: dict[str, Any] = {},
attributes: dict[str, Any] = {},
events: list[TracesEvent] = [],
links: list[TracesLink] = [],
trace_state: str = "",
flags: np.uint32 = 0,
scope: dict[str, Any] = {},
resource_write_mode: Literal["legacy_only", "dual_write"] = "dual_write",
attribute_write_mode: Literal["legacy_only", "dual_write", "json_only"] = "dual_write",
) -> None:
if timestamp is None:
timestamp = datetime.datetime.now()
self.tag_attributes = []
self.attribute_keys = []
self.resource_keys = []
self.span_attributes = []
self.error_events = []
# Calculate ts_bucket_start (30mins bucket)
# Round down to nearest 30-minute interval
minute = timestamp.minute
if minute < 30:
bucket_minute = 0
else:
bucket_minute = 30
bucket_start = timestamp.replace(minute=bucket_minute, second=0, microsecond=0)
self.ts_bucket_start = np.uint64(int(bucket_start.timestamp()))
self.timestamp = timestamp
self.duration_nano = np.uint64(int(duration.total_seconds() * 1e9))
# Initialize trace fields
self.trace_id = trace_id
self.span_id = span_id
self.parent_span_id = parent_span_id
self.trace_state = trace_state
self.flags = flags
self.name = name
self.kind = kind.value
self.kind_string = kind.kind_string()
self.status_code = status_code.value
self.status_message = status_message
self.status_code_string = status_code.name
self.has_error = status_code == TracesStatusCode.STATUS_CODE_ERROR
self.is_remote = self._determine_is_remote(flags)
# Initialize custom fields to empty values
self.response_status_code = ""
self.external_http_url = ""
self.http_url = ""
self.external_http_method = ""
self.http_method = ""
self.http_host = ""
self.db_name = ""
self.db_operation = ""
# Process resources and derive service_name. Spans written before the
# JSON-resource evolution time only populate resources_string (legacy_only);
# spans at or after the evolution time dual-write to both columns.
self.resources_string = {k: str(v) for k, v in resources.items()}
self.resource_json = {} if resource_write_mode == "legacy_only" else dict(self.resources_string)
self.service_name = self.resources_string.get("service.name", "default-service")
for k, v in self.resources_string.items():
self.tag_attributes.append(
TracesTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="resource",
tag_data_type="string",
string_value=v,
number_value=None,
)
)
self.resource_keys.append(TracesResourceOrAttributeKeys(name=k, datatype="string", tag_type="resource"))
self.span_attributes.append(
TracesSpanAttribute(
key=k,
tag_type="resource",
data_type="string",
string_value=v,
)
)
# Calculate resource fingerprint
self.resource_fingerprint = LogsOrTracesFingerprint(self.resources_string).calculate()
# Process scope mirroring the InstrumentationScope on the OTLP span.
scope_name = scope.get("name", "")
scope_version = scope.get("version", "")
scope_string = {k: str(v) for k, v in scope.get("attributes", {}).items()}
self.scope_json = {
"name": scope_name,
"version": scope_version,
"attributes": scope_string,
}
scope_keys = {"scope.name": scope_name, "scope.version": scope_version}
scope_keys.update(scope_string)
for k, v in scope_keys.items():
if v == "":
continue
self.tag_attributes.append(
TracesTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="scope",
tag_data_type="string",
string_value=v,
number_value=None,
)
)
self.attribute_keys.append(TracesResourceOrAttributeKeys(name=k, datatype="string", tag_type="scope"))
# Process attributes by type and populate custom fields
self.attribute_string = {}
self.attributes_number = {}
self.attributes_bool = {}
for k, v in attributes.items():
# Populate custom fields based on attribute keys (following Go exporter logic)
self._populate_custom_attrs(k, v)
if isinstance(v, bool):
self.attributes_bool[k] = v
self.tag_attributes.append(
TracesTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="tag",
tag_data_type="bool",
string_value=None,
number_value=None,
)
)
self.attribute_keys.append(TracesResourceOrAttributeKeys(name=k, datatype="bool", tag_type="tag"))
self.span_attributes.append(
TracesSpanAttribute(
key=k,
tag_type="tag",
data_type="bool",
number_value=None,
)
)
elif isinstance(v, (int, float)):
self.attributes_number[k] = np.float64(v)
self.tag_attributes.append(
TracesTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="tag",
tag_data_type="float64",
string_value=None,
number_value=np.float64(v),
)
)
self.attribute_keys.append(TracesResourceOrAttributeKeys(name=k, datatype="float64", tag_type="tag"))
self.span_attributes.append(
TracesSpanAttribute(
key=k,
tag_type="tag",
data_type="float64",
number_value=np.float64(v),
)
)
else:
self.attribute_string[k] = str(v)
self.tag_attributes.append(
TracesTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="tag",
tag_data_type="string",
string_value=str(v),
number_value=None,
)
)
self.attribute_keys.append(TracesResourceOrAttributeKeys(name=k, datatype="string", tag_type="tag"))
self.span_attributes.append(
TracesSpanAttribute(
key=k,
tag_type="tag",
data_type="string",
string_value=str(v),
)
)
# Spans before the attribute JSON-evolution time populate only the legacy
# attributes_{string,number,bool} maps; spans at or after it dual-write the
# native-typed `attributes` JSON column too, and spans past the map-write
# cutoff populate only the JSON column (metadata rows are still written).
self.attributes_json = {} if attribute_write_mode == "legacy_only" else dict(attributes)
if attribute_write_mode == "json_only":
self.attribute_string, self.attributes_number, self.attributes_bool = {}, {}, {}
# Process events and derive error events. self.events holds the parsed
# response shape; np_arr() encodes back to the DB format on insert.
self.events = []
for event in events:
self.events.append(
{
"name": event.name,
"timeUnixNano": int(event.time_unix_nano),
"attributes": dict(event.attribute_map),
}
)
# Create error events for exception events (following Go exporter logic)
if event.name == "exception":
error_event = self._create_error_event(event)
self.error_events.append(error_event)
# In Python, when you define a function with a mutable default argument (like a list []), that default object is created once when the function is defined,
# not each time the function is called.
# This means all calls to the function share the same default object.
# https://stackoverflow.com/questions/1132941/least-astonishment-in-python-the-mutable-default-argument
links_copy = links.copy() if links else []
if self.parent_span_id != "":
links_copy.insert(
0,
TracesLink(
trace_id=self.trace_id,
span_id=self.parent_span_id,
ref_type=TracesRefType.REF_TYPE_CHILD_OF,
),
)
# self.links holds the parsed response shape (trace_id/span_id only;
# ref_type is dropped to match the API). np_arr() re-encodes for DB insert.
self.links = [{"traceId": link.trace_id, "spanId": link.span_id} for link in links_copy]
self._links_db = json.dumps(
[link.__dict__() for link in links_copy],
separators=(",", ":"),
)
# DB shape per event: {"name", "timeUnixNano", "attributeMap"}. Must match
# what the consume-layer parser in pkg/types/spantypes expects.
self._events_db = [
json.dumps(
{
"name": event.name,
"timeUnixNano": int(event.time_unix_nano),
"attributeMap": dict(event.attribute_map),
},
separators=(",", ":"),
)
for event in events
]
# Initialize resource
self.resource = []
self.resource.append(
TracesResource(
labels=self.resources_string,
fingerprint=self.resource_fingerprint,
seen_at_ts_bucket_start=self.ts_bucket_start,
)
)
def _create_error_event(self, event: TracesEvent) -> TracesErrorEvent:
"""Create error event from exception event (following Go exporter logic)"""
error_id = str(uuid.uuid4()).replace("-", "")
# Create error group ID based on exception type and message
exception_type = event.attribute_map.get("exception.type", "")
exception_message = event.attribute_map.get("exception.message", "")
error_group_content = self.service_name + exception_type + exception_message
error_group_id = hashlib.md5(error_group_content.encode()).hexdigest()
return TracesErrorEvent(
event=event,
error_id=error_id,
error_group_id=error_group_id,
trace_id=self.trace_id,
span_id=self.span_id,
service_name=self.service_name,
resources=self.resources_string,
)
def _determine_is_remote(self, flags: np.uint32) -> str:
"""Determine if span is remote based on flags (following Go exporter logic)"""
has_is_remote_mask = 0x00000100
is_remote_mask = 0x00000200
if flags & has_is_remote_mask != 0:
if flags & is_remote_mask != 0:
return "yes"
return "no"
return "unknown"
def _populate_custom_attrs( # pylint: disable=too-many-branches
self, key: str, value: Any
) -> None:
"""Populate custom attributes based on attribute keys (following Go exporter logic)"""
str_value = str(value)
if key in ["http.status_code", "http.response.status_code"]:
# Handle both string/int http status codes
try:
status_int = int(str_value)
self.response_status_code = str(status_int)
except ValueError:
self.response_status_code = str_value
elif key in ["http.url", "url.full"] and self.kind == 3: # SPAN_KIND_CLIENT
# For client spans, extract hostname for external URL
try:
parsed = urlparse(str_value)
self.external_http_url = parsed.hostname or str_value
self.http_url = str_value
except Exception: # pylint: disable=broad-exception-caught
self.external_http_url = str_value
self.http_url = str_value
elif key in ["http.method", "http.request.method"] and self.kind == 3: # SPAN_KIND_CLIENT
self.external_http_method = str_value
self.http_method = str_value
elif key in ["http.url", "url.full"] and self.kind != 3:
self.http_url = str_value
elif key in ["http.method", "http.request.method"] and self.kind != 3:
self.http_method = str_value
elif key in [
"http.host",
"server.address",
"client.address",
"http.request.header.host",
]:
self.http_host = str_value
elif key in ["db.name", "db.namespace"]:
self.db_name = str_value
elif key in ["db.operation", "db.operation.name"]:
self.db_operation = str_value
elif key == "rpc.grpc.status_code":
# Handle both string/int status code in GRPC spans
try:
status_int = int(str_value)
self.response_status_code = str(status_int)
except ValueError:
self.response_status_code = str_value
elif key == "rpc.jsonrpc.error_code":
self.response_status_code = str_value
def np_arr(self) -> np.array:
return np.array(
[
self.ts_bucket_start,
self.resource_fingerprint,
self.timestamp,
self.trace_id,
self.span_id,
self.trace_state,
self.parent_span_id,
self.flags,
self.name,
self.kind,
self.kind_string,
self.duration_nano,
self.status_code,
self.status_message,
self.status_code_string,
self.attribute_string,
self.attributes_number,
self.attributes_bool,
self.resources_string,
self._events_db,
self._links_db,
self.response_status_code,
self.external_http_url,
self.http_url,
self.external_http_method,
self.http_method,
self.http_host,
self.db_name,
self.db_operation,
self.has_error,
self.is_remote,
self.resource_json,
self.scope_json,
self.attributes_json,
],
dtype=object,
)
@classmethod
def from_dict(
cls,
data: dict,
) -> "Traces":
# parse timestamp from iso format
timestamp = parse_timestamp(data["timestamp"])
duration = parse_duration(data.get("duration", "PT1S"))
kind = TracesKind.from_value(data.get("kind", TracesKind.SPAN_KIND_INTERNAL.value))
status_code = TracesStatusCode.from_value(data.get("status_code", TracesStatusCode.STATUS_CODE_UNSET.value))
return cls(
timestamp=timestamp,
duration=duration,
trace_id=data["trace_id"],
span_id=data["span_id"],
parent_span_id=data.get("parent_span_id", ""),
name=data.get("name", "default span"),
kind=kind,
status_code=status_code,
status_message=data.get("status_message", ""),
resources=data.get("resources", {}),
attributes=data.get("attributes", {}),
trace_state=data.get("trace_state", ""),
flags=data.get("flags", 0),
scope=data.get("scope", {}),
events=[
TracesEvent(
name=event["name"],
timestamp=parse_timestamp(event["timestamp"]),
attribute_map={k: str(v) for k, v in (event.get("attributes") or {}).items()},
)
for event in data.get("events") or []
],
)
@classmethod
def load_from_file(
cls,
file_path: str,
base_time: datetime.datetime | None = None,
) -> list["Traces"]:
"""Load traces from a JSONL file."""
data_list = []
with open(file_path, encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
data_list.append(json.loads(line))
if not data_list:
return []
# If base_time provided, calculate time offset
time_offset = datetime.timedelta(0)
if base_time is not None:
# Find earliest timestamp
earliest = None
for data in data_list:
ts = parse_timestamp(data["timestamp"])
if earliest is None or ts < earliest:
earliest = ts
if earliest is not None:
time_offset = base_time - earliest
traces = []
for data in data_list:
# add time offset to timestamp
original_ts = parse_timestamp(data["timestamp"])
duration = parse_duration(data.get("duration", "PT1S"))
adjusted_ts = original_ts + time_offset
data["timestamp"] = adjusted_ts.isoformat()
# parse duration of the span
data["duration"] = duration
traces.append(cls.from_dict(data))
return traces
def insert_traces_to_clickhouse(conn, traces: list[Traces]) -> None:
"""
Insert traces into ClickHouse tables following the same logic as the Go exporter.
Handles insertion into:
- distributed_signoz_index_v3 (main traces table)
- distributed_traces_v3_resource (resource fingerprints)
- distributed_tag_attributes_v2 (tag attributes)
- distributed_span_attributes_keys (attribute keys)
- distributed_signoz_error_index_v2 (error events)
Pure function so the seeder container (tests/seeder/) can reuse the
exact insert path used by the pytest fixtures. `conn` is a
clickhouse-connect Client.
"""
resources: list[TracesResource] = []
for trace in traces:
resources.extend(trace.resource)
if len(resources) > 0:
conn.insert(
database="signoz_traces",
table="distributed_traces_v3_resource",
data=[resource.np_arr() for resource in resources],
)
tag_attributes: list[TracesTagAttributes] = []
for trace in traces:
tag_attributes.extend(trace.tag_attributes)
if len(tag_attributes) > 0:
conn.insert(
database="signoz_traces",
table="distributed_tag_attributes_v2",
data=[tag_attribute.np_arr() for tag_attribute in tag_attributes],
)
attribute_keys: list[TracesResourceOrAttributeKeys] = []
resource_keys: list[TracesResourceOrAttributeKeys] = []
for trace in traces:
attribute_keys.extend(trace.attribute_keys)
resource_keys.extend(trace.resource_keys)
if len(attribute_keys) > 0:
conn.insert(
database="signoz_traces",
table="distributed_span_attributes_keys",
data=[attribute_key.np_arr() for attribute_key in attribute_keys],
)
if len(resource_keys) > 0:
conn.insert(
database="signoz_traces",
table="distributed_span_attributes_keys",
data=[resource_key.np_arr() for resource_key in resource_keys],
)
conn.insert(
database="signoz_traces",
table="distributed_signoz_index_v3",
column_names=[
"ts_bucket_start",
"resource_fingerprint",
"timestamp",
"trace_id",
"span_id",
"trace_state",
"parent_span_id",
"flags",
"name",
"kind",
"kind_string",
"duration_nano",
"status_code",
"status_message",
"status_code_string",
"attributes_string",
"attributes_number",
"attributes_bool",
"resources_string",
"events",
"links",
"response_status_code",
"external_http_url",
"http_url",
"external_http_method",
"http_method",
"http_host",
"db_name",
"db_operation",
"has_error",
"is_remote",
"resource",
"scope",
"attributes",
],
data=[trace.np_arr() for trace in traces],
)
error_events: list[TracesErrorEvent] = []
for trace in traces:
error_events.extend(trace.error_events)
if len(error_events) > 0:
conn.insert(
database="signoz_traces",
table="distributed_signoz_error_index_v2",
column_names=ERROR_INDEX_COLUMNS,
data=[error_event.np_arr() for error_event in error_events],
)
_TRACES_TABLES_TO_TRUNCATE = [
"signoz_index_v3",
"traces_v3_resource",
"tag_attributes_v2",
"span_attributes_keys",
"signoz_error_index_v2",
"top_level_operations",
"trace_summary",
]
def insert_top_level_operations_to_clickhouse(conn, operations: list[tuple[str, str]]) -> None:
"""Seed distributed_top_level_operations with (name, serviceName) rows so the
isEntryPoint span-scope filter has entries to match against. The `time`
column defaults to now(), which satisfies the filter's `time >= start`
guard for any recent query window."""
if not operations:
return
conn.insert(
database="signoz_traces",
table="distributed_top_level_operations",
column_names=["name", "serviceName"],
data=[[name, service_name] for name, service_name in operations],
)
def truncate_traces_tables(conn, cluster: str) -> None:
"""Truncate all traces tables. Used by the pytest fixture teardown and by
the seeder's DELETE /telemetry/traces endpoint."""
for table in _TRACES_TABLES_TO_TRUNCATE:
conn.query(f"TRUNCATE TABLE signoz_traces.{table} ON CLUSTER '{cluster}' SYNC")
@pytest.fixture(name="insert_traces", scope="function")
def insert_traces(
clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[[list[Traces]], None], Any]:
def _insert_traces(traces: list[Traces]) -> None:
insert_traces_to_clickhouse(clickhouse.conn, traces)
yield _insert_traces
truncate_traces_tables(
clickhouse.conn,
clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"],
)
def insert_attribute_evolution_to_clickhouse(conn, signal: str, release_time: datetime.datetime) -> None:
"""Seed the `attributes` JSON column-evolution row for a signal at release_time. Unlike the
resource row (seeded by the migrator at install), the attribute JSON rollout is install-specific
and not migrator-seeded, so tests insert it to gate map-vs-JSON resolution across a window."""
# insert_deduplicate=0: successive tests seed a byte-identical row (release_time is
# minute-aligned), and ReplicatedMergeTree would drop the re-insert as a duplicate even
# after the prior test's teardown deleted it, leaving the querier to fall back to the Map.
conn.command(
"""
INSERT INTO signoz_metadata.distributed_column_evolution_metadata
(signal, column_name, column_type, field_context, field_name, version, release_time)
SETTINGS insert_deduplicate = 0
VALUES (%(signal)s, 'attributes', 'JSON()', 'attribute', '__all__', 1, %(release_time_ns)s)
""",
parameters={"signal": signal, "release_time_ns": int(release_time.timestamp() * 1e9)},
)
@pytest.fixture(name="seed_attribute_evolution", scope="function")
def seed_attribute_evolution(
clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[[str, datetime.datetime], None], Any]:
def _seed(signal: str, release_time: datetime.datetime) -> None:
insert_attribute_evolution_to_clickhouse(clickhouse.conn, signal, release_time)
yield _seed
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
clickhouse.conn.query(f"ALTER TABLE signoz_metadata.column_evolution_metadata ON CLUSTER '{cluster}' DELETE WHERE column_name = 'attributes' AND field_context = 'attribute' AND field_name = '__all__' SETTINGS mutations_sync = 1")
# A rollout far before any test's query window, so seeding it makes the querier read span
# attributes entirely from the native JSON column rather than straddling into the legacy Map.
ATTRIBUTE_JSON_ROLLOUT_TIME = datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC)
@pytest.fixture(name="use_attribute_backend")
def use_attribute_backend(
seed_attribute_evolution: Callable[[str, datetime.datetime], None],
) -> Callable[[str], None]:
"""Applies the physical attribute layout a test runs under. "map" leaves the querier on the
legacy attributes_{string,number,bool} maps; "json" seeds the column-evolution row so it reads
the native `attributes` JSON column instead. Pair with
@pytest.mark.parametrize("attribute_backend", ["map", "json"]) and call it once in the body;
insert_traces dual-writes both layouts, so a test that passes under both has strict Map/JSON
parity."""
def _apply(backend: str) -> None:
if backend == "json":
seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
return _apply
@pytest.fixture(name="insert_top_level_operations", scope="function")
def insert_top_level_operations(
clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[[list[tuple[str, str]]], None], Any]:
def _insert(operations: list[tuple[str, str]]) -> None:
insert_top_level_operations_to_clickhouse(clickhouse.conn, operations)
yield _insert
clickhouse.conn.query(f"TRUNCATE TABLE signoz_traces.top_level_operations ON CLUSTER '{clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' SYNC")
@pytest.fixture(name="remove_traces_ttl_and_storage_settings", scope="function")
def remove_traces_ttl_and_storage_settings(signoz: types.SigNoz):
"""
Remove any custom TTL settings on traces tables to revert to default retention.
Also resets storage policy to default by recreating tables if needed.
"""
tables = [
"signoz_index_v3",
"traces_v3_resource",
"signoz_error_index_v2",
"usage_explorer",
"dependency_graph_minutes_v2",
"trace_summary",
"span_attributes_keys",
]
for table in tables:
try:
signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_traces.{table} ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' REMOVE TTL")
signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_traces.{table} ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' RESET SETTING storage_policy;")
except Exception as e: # pylint: disable=broad-exception-caught
print(f"ttl and storage policy reset failed for {table}: {e}")
# ============================================================================
# Clean / corrupt factor (shared across the traces querier tests)
# ============================================================================
# Colliding/corrupt metadata mixed into every seeded span in the "corrupt"
# variant of the list/aggregation tests. It injects:
# - intrinsic column names as span attributes (timestamp, duration_nano, ...)
# - calculated column names as span attributes (http_method, db_name, ...)
# - a key (service.name) present in both attributes and resources
# Values deliberately mix Python types (str / int / float / bool) so the
# collision also spans the attributes_string / attributes_number /
# attributes_bool type-variant columns — e.g. a string attribute named
# duration_nano vs the numeric intrinsic, a numeric attribute named
# response_status_code vs the string calculated column, a bool attribute named
# db_name vs the string calculated column.
# Field-key collision resolution must keep the real intrinsic/calculated/resource
# columns winning regardless of the attribute's type, so every assertion holds
# identically for clean and corrupt spans. The corrupt values only ever surface
# inside the raw `attributes` map.
CORRUPT_ATTRIBUTES: dict[str, Any] = {
# intrinsic names (real column type in comment)
"timestamp": "corrupt_data", # string vs number
"trace_id": 12345, # number vs string
"span_id": True, # bool vs string
"parent_span_id": "corrupt_data", # string vs string
"trace_state": 99, # number vs string
"flags": "corrupt_data", # string vs number
"name": 42, # number vs string
"kind": "corrupt_data", # string vs number
"kind_string": 7, # number vs string
"duration_nano": "corrupt_data", # string vs number
"status_code": True, # bool vs number
"status_message": 500, # number vs string
"status_code_string": False, # bool vs string
# calculated names (real column type in comment)
"response_status_code": 999, # number vs string
"external_http_url": 8080, # number vs string
"http_url": True, # bool vs string
"external_http_method": 1, # number vs string
"http_method": False, # bool vs string
"http_host": 12, # number vs string
"db_name": True, # bool vs string
"db_operation": 3.5, # number vs string
"has_error": "corrupt_data", # string vs bool
"is_remote": 1, # number vs string
# collides with the resource key of the same name
"service.name": "collision-attr-value",
}
# Resources are always stored as strings, so the resource-side collision only
# varies the key names.
CORRUPT_RESOURCES: dict[str, Any] = {
"timestamp": "corrupt_data",
"duration_nano": "corrupt_data",
"http_method": "corrupt_data",
}
# A TYPE-CONSISTENT collision, distinct from CORRUPT_* (whose wrong-type values are
# dropped by field-key resolution): a numeric span attribute named `duration_nano`
# shares both the name AND a compatible type with the intrinsic duration_nano (UInt64)
# column, so resolution unions it with the column into a multiIf. This exercises the
# collision path that regressed with ClickHouse NO_COMMON_TYPE (386) — the intrinsic
# column must still win. Kept out of CORRUPT_ATTRIBUTES because a type-consistent
# collision changes raw-select output (the multiIf stringifies the value), which the
# list tests assert against; only aggregation/filter tests opt into this variant.
COLLISION_ATTRIBUTES: dict[str, Any] = {
"duration_nano": 1.0, # numeric attr vs the numeric intrinsic (type-consistent)
}
def trace_noise(variant: str) -> tuple[dict[str, Any], dict[str, Any]]:
"""(extra_attributes, extra_resources) to merge into every span a traces test seeds, keyed by
variant. "clean" adds nothing; "corrupt" injects wrong-type colliding intrinsic/calculated field
names (dropped by resolution, so results are unchanged); "collision" injects a type-consistent
numeric duration_nano attribute that unions into a multiIf, exercising collision resolution on
aggregation/filter paths. Returns fresh dicts so callers can mutate them safely."""
if variant == "clean":
return {}, {}
if variant == "collision":
return dict(COLLISION_ATTRIBUTES), {}
return dict(CORRUPT_ATTRIBUTES), dict(CORRUPT_RESOURCES)